@jant/core 0.6.16 → 0.7.1

package/README.md

@@ -21,7 +21,7 @@ npm run dev
 For a traditional server deployment, use the official Docker image:
 
 - [`owenyoung/jant`](https://hub.docker.com/r/owenyoung/jant)
-- [Docker deployment guide](https://github.com/jant-me/jant/blob/main/docs/deployment-docker.md)
+- [Docker deployment guide](https://jant.me/docs/deployment-docker)
 
 ## Tech Stack
 
@@ -35,16 +35,16 @@ For a traditional server deployment, use the official Docker image:
 
 ## Documentation
 
-- [Overview](https://github.com/jant-me/jant/blob/main/docs/overview.md)
-- [Getting Started](https://github.com/jant-me/jant/blob/main/docs/getting-started.md)
-- [Writing and Organizing Posts](https://github.com/jant-me/jant/blob/main/docs/writing-and-organizing.md)
-- [Deploy on Cloudflare](https://github.com/jant-me/jant/blob/main/docs/deployment.md)
-- [Deploy with Docker](https://github.com/jant-me/jant/blob/main/docs/deployment-docker.md)
-- [Configuration](https://github.com/jant-me/jant/blob/main/docs/configuration.md)
-- [Export and Import](https://github.com/jant-me/jant/blob/main/docs/export-and-import.md)
-- [Backups and Recovery](https://github.com/jant-me/jant/blob/main/docs/backups.md)
-- [Theming](https://github.com/jant-me/jant/blob/main/docs/theming.md)
-- [API Reference](https://github.com/jant-me/jant/blob/main/docs/API.md)
+- [Overview](https://jant.me/docs)
+- [Getting Started](https://jant.me/docs/getting-started)
+- [Writing and Organizing Posts](https://jant.me/docs/writing-and-organizing)
+- [Deploy on Cloudflare](https://jant.me/docs/deployment)
+- [Deploy with Docker](https://jant.me/docs/deployment-docker)
+- [Configuration](https://jant.me/docs/configuration)
+- [Export and Import](https://jant.me/docs/export-and-import)
+- [Backups and Recovery](https://jant.me/docs/backups)
+- [Theming](https://jant.me/docs/theming)
+- [API Reference](https://jant.me/docs/api)
 
 ## Contributing
 

package/bin/commands/assets/upload.js

@@ -6,14 +6,25 @@
  * Uses the same S3_* environment variables as media storage. Files with
  * content-hashed names (JS, fonts) are skipped if already present in the
  * bucket — they are immutable. Non-hashed files (e.g. CSS in older builds)
- * are always uploaded.
+ * are always uploaded. An object stored without the compression it should
+ * have is re-uploaded even though its key exists, so a change to the
+ * compression policy reaches assets whose content never changed.
+ *
+ * Text assets are uploaded brotli-compressed at maximum quality, with
+ * `Content-Encoding: br`. The CDN in front of the bucket compresses at a fixed
+ * low quality — measured as brotli q4, which is worse than its own gzip — and
+ * offers no way to raise it, so the only route to q11 is to compress here and
+ * store the result. A client that cannot take brotli is unaffected: the edge
+ * decompresses and re-encodes for it (verified against R2 behind a Cloudflare
+ * custom domain: `br` is passed through, `gzip` is re-encoded, `identity` is
+ * decompressed, all three byte-identical after decoding).
  *
  * Intended CI order: build → upload-assets → deploy container
  */
 
 import { readdir, readFile, stat } from "node:fs/promises";
 import { join, relative, extname } from "node:path";
-import { createReadStream } from "node:fs";
+import { brotliCompressSync, constants } from "node:zlib";
 import { parseArgs } from "node:util";
 import { fileURLToPath } from "node:url";
 import { dirname, resolve } from "node:path";
@@ -41,6 +52,59 @@ function getContentType(filePath) {
   );
 }
 
+/**
+ * Extensions worth pre-compressing. Fonts (woff2), images and icons already
+ * carry their own compression, and running brotli over them costs build time
+ * for nothing — usually for a slightly larger file.
+ */
+const COMPRESSIBLE_EXTENSIONS = new Set([
+  ".js",
+  ".css",
+  ".json",
+  ".map",
+  ".svg",
+  ".txt",
+  ".html",
+]);
+
+/**
+ * Brotli-compress an asset for storage, at a quality only a build can afford.
+ *
+ * Quality 11 is far too slow to run per request, which is why CDNs compress
+ * lower. Here it runs once per file per release, and every visitor gets the
+ * result.
+ *
+ * @param {string} filePath - Source path, used only for its extension.
+ * @param {Buffer} body - The file's bytes.
+ * @returns {Buffer | null} The compressed body, or `null` when the type is not
+ * worth compressing or compression failed to shrink it — upload `body` as-is
+ * and omit `Content-Encoding` in that case.
+ * @example
+ * const packed = compressForUpload("client.css", body);
+ * // packed.length < body.length, or null for a .woff2
+ */
+export function compressForUpload(filePath, body) {
+  if (!COMPRESSIBLE_EXTENSIONS.has(extname(filePath).toLowerCase())) {
+    return null;
+  }
+  const compressed = brotliCompressSync(body, {
+    params: {
+      [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY,
+      [constants.BROTLI_PARAM_SIZE_HINT]: body.length,
+    },
+  });
+  // Tiny or already-dense files can come back bigger. Storing that would make
+  // every request pay for the encoding header and gain nothing.
+  return compressed.length < body.length ? compressed : null;
+}
+
+/** Human-readable byte count for the upload log. */
+function formatBytes(bytes) {
+  return bytes >= 1024 * 1024
+    ? `${(bytes / 1024 / 1024).toFixed(1)} MB`
+    : `${(bytes / 1024).toFixed(1)} KB`;
+}
+
 /**
  * Resolve the default source directory: packages/core/dist/client/_assets
  * relative to this file's location in packages/core/bin/commands/assets/.
@@ -64,8 +128,12 @@ async function walkDir(dir) {
 }
 
 async function loadS3(config) {
-  const { S3Client, PutObjectCommand, ListObjectsV2Command } =
-    await import("@aws-sdk/client-s3");
+  const {
+    S3Client,
+    PutObjectCommand,
+    ListObjectsV2Command,
+    HeadObjectCommand,
+  } = await import("@aws-sdk/client-s3");
   const forcePathStyle = !config.endpoint.includes("amazonaws.com");
   const client = new S3Client({
     endpoint: config.endpoint,
@@ -80,6 +148,7 @@ async function loadS3(config) {
     client,
     PutObjectCommand,
     ListObjectsV2Command,
+    HeadObjectCommand,
     bucket: config.bucket,
   };
 }
@@ -107,6 +176,53 @@ async function listExistingKeys(s3, prefix) {
   return keys;
 }
 
+/**
+ * Of the keys that already exist, the ones stored without the compression they
+ * should have.
+ *
+ * Objects are content-addressed, so a key that exists is assumed to hold the
+ * right bytes and is skipped. That assumption breaks when the *encoding*
+ * changes rather than the content: every asset uploaded before compression was
+ * introduced keeps its key, keeps being skipped, and never picks it up. The
+ * CDN then compresses those at its own low quality on every request, which is
+ * the whole thing compression was meant to avoid.
+ *
+ * Only worth asking about types we would compress, and only when compression
+ * is on. A file that brotli cannot shrink is stored raw on purpose, so its
+ * missing `Content-Encoding` is correct and must not read as stale — otherwise
+ * it would be re-uploaded on every run, forever.
+ *
+ * @param {object} s3 - The client bundle from `loadS3`.
+ * @param {Array<{filePath: string, key: string}>} candidates - Local files
+ * whose key already exists in the bucket.
+ * @returns {Promise<Set<string>>} Keys to upload again.
+ * @example
+ * const stale = await findStaleEncodingKeys(s3, existing);
+ * // Set { "_assets/client-cjk-B7Z0snDu.css" }
+ */
+async function findStaleEncodingKeys(s3, candidates) {
+  const { client, HeadObjectCommand, bucket } = s3;
+  const stale = new Set();
+  const CONCURRENCY = 25;
+
+  for (let i = 0; i < candidates.length; i += CONCURRENCY) {
+    await Promise.all(
+      candidates.slice(i, i + CONCURRENCY).map(async ({ filePath, key }) => {
+        const head = await client.send(
+          new HeadObjectCommand({ Bucket: bucket, Key: key }),
+        );
+        if (head.ContentEncoding === "br") return;
+        // Read it only now: the answer is only interesting for the few objects
+        // that are missing the encoding.
+        const body = await readFile(filePath);
+        if (compressForUpload(filePath, body) !== null) stale.add(key);
+      }),
+    );
+  }
+
+  return stale;
+}
+
 export async function run(argv) {
   const { values } = parseArgs({
     args: argv,
@@ -115,6 +231,8 @@ export async function run(argv) {
       prefix: { type: "string", default: "_assets" },
       "source-dir": { type: "string" },
       "dry-run": { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+      "no-compress": { type: "boolean", default: false },
     },
   });
 
@@ -135,6 +253,12 @@ export async function run(argv) {
     console.log(
       "      --dry-run             Print what would be uploaded without uploading",
     );
+    console.log(
+      "      --force               Re-upload every key, including ones already compressed",
+    );
+    console.log(
+      "      --no-compress         Upload raw bytes instead of brotli-compressed",
+    );
     process.exit(0);
   }
 
@@ -171,6 +295,8 @@ export async function run(argv) {
   const sourceDir = values["source-dir"] ?? resolveDefaultSourceDir();
   const prefix = values.prefix.replace(/^\/+|\/+$/g, "");
   const dryRun = values["dry-run"];
+  const force = values.force;
+  const compress = !values["no-compress"];
 
   // Verify source directory exists
   try {
@@ -181,10 +307,12 @@ export async function run(argv) {
     process.exit(1);
   }
 
-  console.log(`Source:  ${sourceDir}`);
-  console.log(`Bucket:  ${bucket}`);
-  console.log(`Prefix:  ${prefix}`);
-  if (dryRun) console.log("Dry run: no files will be uploaded");
+  console.log(`Source:   ${sourceDir}`);
+  console.log(`Bucket:   ${bucket}`);
+  console.log(`Prefix:   ${prefix}`);
+  console.log(`Encoding: ${compress ? "brotli (quality 11)" : "raw"}`);
+  if (force) console.log("Force:    re-uploading keys that already exist");
+  if (dryRun) console.log("Dry run:  no files will be uploaded");
   console.log("");
 
   const s3 = await loadS3({
@@ -205,19 +333,57 @@ export async function run(argv) {
   let skipped = 0;
 
   const toUpload = [];
+  const alreadyThere = [];
   for (const filePath of files) {
     const relPath = relative(sourceDir, filePath).replace(/\\/g, "/");
     const key = prefix ? `${prefix}/${relPath}` : relPath;
-    if (existingKeys.has(key)) {
-      skipped++;
+    if (existingKeys.has(key) && !force) {
+      alreadyThere.push({ filePath, key });
     } else {
       toUpload.push({ filePath, key });
     }
   }
 
+  // A key that exists is normally left alone, but that skips objects stored
+  // before compression was introduced — they would keep their old encoding
+  // forever. Re-upload those, and only those.
+  let restored = 0;
+  if (compress && alreadyThere.length > 0) {
+    process.stdout.write("Checking stored encodings... ");
+    const stale = await findStaleEncodingKeys(s3, alreadyThere);
+    for (const entry of alreadyThere) {
+      if (stale.has(entry.key)) toUpload.push(entry);
+      else skipped++;
+    }
+    restored = stale.size;
+    console.log(
+      stale.size === 0
+        ? "all compressed"
+        : `${stale.size} stored uncompressed, re-uploading`,
+    );
+  } else {
+    skipped += alreadyThere.length;
+  }
+
+  // Bytes as stored, against bytes as built — what the compression bought.
+  let rawTotal = 0;
+  let storedTotal = 0;
+
+  /** Read a file and decide how it should be stored. */
+  async function prepare(filePath) {
+    const body = await readFile(filePath);
+    const packed = compress ? compressForUpload(filePath, body) : null;
+    rawTotal += body.length;
+    storedTotal += (packed ?? body).length;
+    return { body: packed ?? body, encoded: packed !== null };
+  }
+
   if (dryRun) {
-    for (const { key } of toUpload) {
-      console.log(`  [dry-run] upload ${key}`);
+    for (const { filePath, key } of toUpload) {
+      const { body, encoded } = await prepare(filePath);
+      console.log(
+        `  [dry-run] upload ${key} (${formatBytes(body.length)}${encoded ? ", br" : ""})`,
+      );
     }
     uploaded = toUpload.length;
   } else {
@@ -226,16 +392,17 @@ export async function run(argv) {
       const batch = toUpload.slice(i, i + CONCURRENCY);
       await Promise.all(
         batch.map(async ({ filePath, key }) => {
-          const body = await readFile(filePath);
+          const { body, encoded } = await prepare(filePath);
           const command = new s3.PutObjectCommand({
             Bucket: s3.bucket,
             Key: key,
             Body: body,
             ContentType: getContentType(filePath),
             CacheControl: "public, max-age=31536000, immutable",
+            ...(encoded ? { ContentEncoding: "br" } : {}),
           });
           await s3.client.send(command);
-          process.stdout.write(`  uploaded ${key}\n`);
+          process.stdout.write(`  uploaded ${key}${encoded ? " (br)" : ""}\n`);
           uploaded++;
         }),
       );
@@ -243,5 +410,15 @@ export async function run(argv) {
   }
 
   console.log("");
-  console.log(`Done. ${uploaded} uploaded, ${skipped} skipped.`);
+  console.log(
+    `Done. ${uploaded} uploaded, ${skipped} skipped` +
+      `${restored > 0 ? `, ${restored} re-uploaded for compression` : ""}.`,
+  );
+  if (compress && rawTotal > 0) {
+    const saved = rawTotal - storedTotal;
+    console.log(
+      `Stored ${formatBytes(storedTotal)} of ${formatBytes(rawTotal)} built ` +
+        `(${formatBytes(saved)} saved, ${((saved / rawTotal) * 100).toFixed(1)}%).`,
+    );
+  }
 }

package/bin/commands/export.js

@@ -1,7 +1,7 @@
 import { writeFileSync } from "node:fs";
 import { resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { queryD1 } from "../lib/d1-query.js";
+import { D1_DUMP_PAGE_SIZE, queryD1 } from "../lib/d1-query.js";
 import { openNodeDatabase } from "../lib/node-database.js";
 import { dumpDatabaseToSql } from "../lib/sql-export.js";
 import {
@@ -108,6 +108,8 @@ export async function run(argv) {
       {
         source: runtime === "d1-remote" ? "remote" : "local",
         dialect: "sqlite",
+        // D1 answers through Wrangler's buffered output; read it in pages.
+        pageSize: D1_DUMP_PAGE_SIZE,
       },
     );
   }