neme-world 0.1.0 → 0.2.0

package/README.md

@@ -43,9 +43,13 @@ npx neme-world publish     # uploads your build and puts it live
 
 ## What publish does
 
-It zips `dir` and uploads it as a new version, which goes live at once. Hidden files (anything starting with a
-dot) and `node_modules` are never uploaded. Every version is kept, and you can make any earlier one live again
-from your dashboard.
+It uploads `dir` file by file as a new version, which goes live at once. Only files that changed since your last
+publish are sent, so updating a large game is quick. Hidden files (anything starting with a dot) and
+`node_modules` are never uploaded. Every version is kept, and you can make any earlier one live again from your
+dashboard.
+
+A single file can be up to 95 MB, and a game up to 1 GB. One publish per game runs at a time: starting a second
+replaces the first.
 
 ## Options
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neme-world",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Publish web games to neme.world from the command line",
   "keywords": ["neme", "games", "web games", "html5 games", "publish", "deploy", "cli"],
   "homepage": "https://neme.world/developers",
@@ -9,8 +9,5 @@
   "type": "module",
   "bin": { "neme": "src/cli.js" },
   "files": ["src/cli.js"],
-  "engines": { "node": ">=20" },
-  "dependencies": {
-    "fflate": "^0.8.3"
-  }
+  "engines": { "node": ">=20" }
 }

package/src/cli.js

@@ -8,11 +8,11 @@
  *
  * Plain JavaScript on purpose: it runs from a checkout or from npm with no build.
  */
-import { realpathSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { createReadStream, openAsBlob, realpathSync } from 'node:fs'
 import { readdir, readFile, stat, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { parseArgs } from 'node:util'
-import { zipSync } from 'fflate'
 
 const DEFAULT_API = 'https://neme.world'
 const SKIP_DIRS = new Set(['node_modules'])
@@ -56,25 +56,43 @@ export async function readManifest(dir) {
 }
 
 /**
- * Collects every publishable file under `root`. Dotfiles and node_modules are
- * left out so secrets and dependencies don't get uploaded by accident.
+ * @typedef {object} LocalFile
+ * @property {string} path    Path inside the build, with forward slashes.
+ * @property {string} file    Where it is on disk.
+ * @property {number} size
+ * @property {string} sha256
+ */
+
+/** @param {string} file @returns {Promise<string>} */
+function sha256Of(file) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash('sha256')
+    createReadStream(file).on('data', (chunk) => hash.update(chunk)).on('error', reject).on('end', () => resolve(hash.digest('hex')))
+  })
+}
+
+/**
+ * Lists every publishable file under `root`, with its size and hash. Dotfiles
+ * and node_modules are left out so secrets and dependencies don't get uploaded
+ * by accident. Files are hashed from disk, never held in memory whole.
  * @param {string} root
- * @returns {Promise<Record<string, Uint8Array>>}
+ * @returns {Promise<LocalFile[]>}
  */
 export async function collectFiles(root) {
-  /** @type {Record<string, Uint8Array>} */
-  const files = {}
+  /** @type {LocalFile[]} */
+  const files = []
   /** @param {string} rel */
   async function walk(rel) {
     for (const entry of await readdir(path.join(root, rel), { withFileTypes: true })) {
       if (entry.name.startsWith('.') || SKIP_DIRS.has(entry.name)) continue
       const child = rel ? `${rel}/${entry.name}` : entry.name
+      const file = path.join(root, child)
       if (entry.isDirectory()) await walk(child)
-      else if (entry.isFile() && child !== 'neme.json') files[child] = await readFile(path.join(root, child))
+      else if (entry.isFile() && child !== 'neme.json') files.push({ path: child, file, size: (await stat(file)).size, sha256: await sha256Of(file) })
     }
   }
   await walk('')
-  return files
+  return files.sort((a, b) => (a.path < b.path ? -1 : 1))
 }
 
 /**
@@ -82,7 +100,7 @@ export async function collectFiles(root) {
  * @param {string} token
  * @param {string} method
  * @param {string} route
- * @param {{ json?: unknown, body?: Uint8Array }} [o]
+ * @param {{ json?: unknown, body?: Blob }} [o]
  */
 async function call(api, token, method, route, o = {}) {
   /** @type {Record<string, string>} */
@@ -93,8 +111,8 @@ async function call(api, token, method, route, o = {}) {
     headers['content-type'] = 'application/json'
     body = JSON.stringify(o.json)
   } else if (o.body) {
-    headers['content-type'] = 'application/zip'
-    body = /** @type {BodyInit} */ (o.body)
+    headers['content-type'] = 'application/octet-stream'
+    body = o.body
   }
   let res
   try {
@@ -102,7 +120,7 @@ async function call(api, token, method, route, o = {}) {
   } catch (err) {
     throw new CliError(`Could not reach ${api}: ${/** @type {Error} */ (err).cause ?? /** @type {Error} */ (err).message}`)
   }
-  const data = /** @type {any} */ (await res.json().catch(() => ({})))
+  const data = /** @type {any} */ (res.status === 204 ? {} : await res.json().catch(() => ({})))
   if (!res.ok) throw new CliError(`${data.error ?? `Request failed (${res.status})`}`)
   return data
 }
@@ -110,9 +128,14 @@ async function call(api, token, method, route, o = {}) {
 /** @param {number} n */
 const size = (n) => (n < 1048576 ? `${(n / 1024).toFixed(1)} KB` : `${(n / 1048576).toFixed(1)} MB`)
 
+/** How many files are sent at once. */
+const PARALLEL = 6
+
 /**
+ * Publishes file by file: declare everything, send only what the platform does
+ * not already have, then turn the upload into a version.
  * @param {{ dir: string, api: string, token: string, log?: (line: string) => void }} o
- * @returns {Promise<{ version: number, url: string, warnings: string[] }>}
+ * @returns {Promise<{ version: number, url: string, warnings: string[], uploaded: number }>}
  */
 export async function publish({ dir, api, token, log = console.log }) {
   const manifest = await readManifest(dir)
@@ -122,22 +145,50 @@ export async function publish({ dir, api, token, log = console.log }) {
   const info = await stat(buildDir).catch(() => null)
   if (!info?.isDirectory()) throw new CliError(`Build folder ${buildDir} does not exist. Did you run your build?`)
   const files = await collectFiles(buildDir)
-  if (!files[entry]) throw new CliError(`${buildDir} has no ${entry}. Set "dir" in neme.json to your build output folder.`)
-  if (manifest.cover && !files[manifest.cover]) throw new CliError(`Cover "${manifest.cover}" was not found in ${buildDir}.`)
+  if (!files.some((f) => f.path === entry)) throw new CliError(`${buildDir} has no ${entry}. Set "dir" in neme.json to your build output folder.`)
+  if (manifest.cover && !files.some((f) => f.path === manifest.cover)) throw new CliError(`Cover "${manifest.cover}" was not found in ${buildDir}.`)
 
-  const count = Object.keys(files).length
-  const zip = zipSync(files, { level: 6 })
-  log(`${manifest.title}: ${count} file${count === 1 ? '' : 's'}, ${size(zip.length)} zipped`)
+  const total = files.reduce((n, f) => n + f.size, 0)
+  log(`${manifest.title}: ${files.length} file${files.length === 1 ? '' : 's'}, ${size(total)}`)
 
   const { slug, dir: _dir, entry: _entry, ...details } = manifest
   await call(api, token, 'PUT', `/games/${slug}`, { json: details })
-  const out = await call(api, token, 'POST', `/games/${slug}/versions?entry=${encodeURIComponent(entry)}`, { body: zip })
+  const begun = await call(api, token, 'POST', `/games/${slug}/uploads`, {
+    json: { entry, files: files.map(({ path, size, sha256 }) => ({ path, size, sha256 })) },
+  })
+
+  /** @type {{ sha256: string, size: number }[]} */
+  const missing = begun.missing
+  const byHash = new Map(files.map((f) => [f.sha256, f]))
+  const sending = missing.reduce((n, m) => n + m.size, 0)
+  if (missing.length) log(`  uploading ${missing.length} file${missing.length === 1 ? '' : 's'} (${size(sending)}); ${files.length - missing.length} already there`)
+  else log('  nothing new to upload')
+
+  const queue = [...missing]
+  const send = async () => {
+    for (let next = queue.pop(); next; next = queue.pop()) {
+      const local = /** @type {LocalFile} */ (byHash.get(next.sha256))
+      const route = `/games/${slug}/uploads/${begun.upload}/blobs/${next.sha256}`
+      try {
+        await call(api, token, 'PUT', route, { body: await openAsBlob(local.file) })
+      } catch (err) {
+        // One retry covers a dropped connection; anything else is reported with the file's name.
+        try {
+          await call(api, token, 'PUT', route, { body: await openAsBlob(local.file) })
+        } catch (again) {
+          throw new CliError(`Could not upload ${local.path}: ${/** @type {Error} */ (again).message}`)
+        }
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: PARALLEL }, send))
 
-  for (const w of out.warnings ?? []) log(`  warning: ${w}`)
+  const out = await call(api, token, 'POST', `/games/${slug}/uploads/${begun.upload}/complete`, { json: {} })
+  for (const w of [...(begun.warnings ?? []), ...(out.warnings ?? [])]) log(`  warning: ${w}`)
   const saved = out.transferBytes < out.bytes ? ` (${size(out.bytes)} before compression)` : ''
   log(`  v${out.version} is live — ${size(out.transferBytes)} to download${saved}`)
   log(`  ${out.game.url}`)
-  return { version: out.version, url: out.game.url, warnings: out.warnings ?? [] }
+  return { version: out.version, url: out.game.url, warnings: out.warnings ?? [], uploaded: missing.length }
 }
 
 /** @param {string} dir */