@xgamesfun/xcore 0.1.0 → 0.2.1

package/lib/dev/platform.mjs

@@ -4,14 +4,15 @@
 //   - the game's server, through ./gf-local.js's HOST binding (handleHost),
 //   - the page, through the dev host's /platform/v1/* routes (handlePlatform, below).
 // Kept in <game>/.gamesfun/dev-platform.json (a hidden folder: never published), written a moment after each change.
+// The game's project files (env.GF.files) are ./files.mjs's, in <game>/.gamesfun/dev/files/.
 //
-// Every answer has games.fun's shape: {ok: true, …} or {ok: false, code, why}. A refusal is an answer, never a throw,
+// Every answer has xgames.fun's shape: {ok: true, …} or {ok: false, code, why}. A refusal is an answer, never a throw,
 // so a game's server reads it the same here as hosted. The limits are the hosted ones (PLATFORM_LIMITS), so a game
 // that stays inside them here stays inside them there; the per-minute rate limits are left out on this machine.
 //
-// Local sign-in. No games.fun signs a player in on this machine, so the dev host hands out its own tokens
+// Local sign-in. No xgames.fun signs a player in on this machine, so the dev host hands out its own tokens
 // (POST /platform/v1/dev/token): `gfdev1.<payload>.<HMAC-SHA256>`, signed with a key made when it starts and never
-// written anywhere, for this game only, lasting an hour. verifyPlayer accepts exactly those; a games.fun token (EdDSA,
+// written anywhere, for this game only, lasting an hour. verifyPlayer accepts exactly those; an xgames.fun token (EdDSA,
 // from the play host) is refused here with words that say so.
 import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
 import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
@@ -22,9 +23,14 @@ import {
   SAVE_KEY,
   SERVER_FORMAT,
   checkLootResult,
+  FILES,
 } from '../server/contract.mjs';
+import { createFileStore } from './files.mjs';
+import { stateFolder } from '../state-folder.mjs';
 
 const TOKEN_MS = 60 * 60 * 1000;
+/** How much of a stream with no declared length GF.files.put reads (xgames.fun's UNSIZED_MAX); a longer one needs it. */
+const UNSIZED_MAX = 1024 * 1024;
 // eslint-disable-next-line no-control-regex -- it exists to drop control characters
 const UNSAFE = /[\u0000-\u001f<>]/g;
 const NAME = s =>
@@ -37,7 +43,7 @@ const okPlayer = p => typeof p === 'string' && p.length >= 1 && p.length <= L.sa
 
 /**
  * One game's platform store. → {slug, gameId, run(op, args), handleHost(request), devToken({name, id}),
- * verifyPlayer(token), flush(), file}
+ * verifyPlayer(token), flush(), file, files (./files.mjs), setFilesOrigin(origin)}
  */
 export function createPlatformStore({
   dir,
@@ -48,11 +54,21 @@ export function createPlatformStore({
 } = {}) {
   const key = randomBytes(32);
   const gameId = `dev-${slug}`.slice(0, 60);
+  // the public files' origin: the dev host's files server, once it listens
+  let filesOrigin = null;
+  const files = createFileStore({ dir, origin: () => filesOrigin });
   let data = null,
+    loading = null,
     timer = null,
     writing = Promise.resolve();
-  const load = async () => {
-    if (data) return data;
+  // the file is read once: calls that come at once before it is read all wait on that one read, so none of them puts
+  // a fresh copy of the file over what another has changed already
+  const load = () =>
+    (loading ??= readStore().catch(e => {
+      loading = null; // read again next time
+      throw e;
+    }));
+  const readStore = async () => {
     try {
       data = JSON.parse(await readFile(file, 'utf8'));
     } catch {
@@ -72,6 +88,7 @@ export function createPlatformStore({
     return data;
   };
   const write = async () => {
+    if (dir && path.dirname(file) === path.join(dir, '.gamesfun')) await stateFolder(dir);
     await mkdir(path.dirname(file), { recursive: true });
     await writeFile(`${file}.tmp`, JSON.stringify(data, null, 2) + '\n');
     await rename(`${file}.tmp`, file);
@@ -99,7 +116,7 @@ export function createPlatformStore({
       expires: exp * 1000,
     };
   }
-  /** env.GF.verifyPlayer, on this machine: games.fun's answer shape (no name: hosted it has none), plus `dev: true`. */
+  /** env.GF.verifyPlayer, on this machine: xgames.fun's answer shape (no name: hosted it has none), plus `dev: true`. */
   function verifyPlayer(token) {
     const { name: _name, ...v } = readToken(token);
     return v;
@@ -111,7 +128,7 @@ export function createPlatformStore({
     if (parts[0] !== 'gfdev1' || parts.length !== 3)
       return no(
         'bad',
-        'In xcore dev only its own local sign-in tokens are checked (GamesFun.signIn() gets one); a games.fun token is checked when the game is hosted.',
+        'In xcore dev only its own local sign-in tokens are checked (GamesFun.signIn() gets one); an xgames.fun token is checked when the game is hosted.',
       );
     const want = Buffer.from(sign(`gfdev1.${parts[1]}`)),
       got = Buffer.from(parts[2]);
@@ -213,7 +230,7 @@ export function createPlatformStore({
       persist();
       return { ok: true, improved, best: row.score, rank: rankOf(b, row.score) };
     },
-    /** The board's settings, made now when new: {order?, writer?: 'server' | 'any'} (games.fun's GF.scores.setup).
+    /** The board's settings, made now when new: {order?, writer?: 'server' | 'any'} (xgames.fun's GF.scores.setup).
      * `board` may be a pattern (`daily-*`, `*`): then only the rule is kept, for those boards now and to come. */
     async 'scores.setup'(board, { order = 'high', writer } = {}) {
       const d = await load();
@@ -280,7 +297,7 @@ export function createPlatformStore({
       if (d.reports.length > 200) d.reports.shift();
       persist();
       log(
-        `loot.report ${entry.id} (on this machine nothing is signed or paid; hosted, games.fun's keeper signs it with the game's host key; a voucher key of the server's own, xcore add vouchers, signs here too): ${JSON.stringify(result).slice(0, 300)}`,
+        `loot.report ${entry.id} (on this machine nothing is signed or paid; hosted, xgames.fun's keeper signs it with the game's host key; a voucher key of the server's own, xcore add vouchers, signs here too): ${JSON.stringify(result).slice(0, 300)}`,
       );
       return { ok: true, result: null, vouchers: [], local: true, id: entry.id };
     },
@@ -293,6 +310,10 @@ export function createPlatformStore({
     async info() {
       return { ok: true, contract: SERVER_FORMAT, gameId, slug, local: true };
     },
+    'files.head': p => files.head(p),
+    'files.list': opts => files.list(opts ?? {}),
+    'files.delete': p => files.delete(p),
+    'files.usage': () => files.usage(),
   };
 
   /** The board's writer: its own rule, else the longest pattern (`daily-*`, `*`) its name starts with, else 'any'. */
@@ -360,7 +381,54 @@ export function createPlatformStore({
   }
   /** gf-local.js's HOST binding: POST http://gf.local/<op> [args…] → the answer as JSON. */
   async function handleHost(request) {
-    const op = new URL(request.url).pathname.slice(1);
+    const url = new URL(request.url);
+    const op = url.pathname.slice(1);
+    // a file's bytes travel as they are: the put's body, the get's answer (its facts in x-gf-answer)
+    if (op === 'files.put') {
+      const q = url.searchParams;
+      // a stream: held to exactly its declared length, or read up to 1 MB when it has none (xgames.fun's put does so)
+      const stream = q.get('stream') === '1';
+      const length = q.has('length') ? Number(q.get('length')) : null;
+      if (stream && length !== null && length > FILES.fileBytes)
+        return Response.json({ ok: false, code: 'too-big', why: `A file may be ${FILES.fileBytes / 1048576} MB.` });
+      const cap = !stream ? FILES.fileBytes : length !== null ? length : UNSIZED_MAX;
+      const body = await readCapped(request, cap);
+      if (!body) {
+        if (stream && length === null)
+          return Response.json({
+            ok: false,
+            code: 'body',
+            why: `A stream over ${UNSIZED_MAX / 1048576} MB needs its length: put(path, stream, {length}). (A string or an ArrayBuffer needs none.)`,
+          });
+        if (stream)
+          return Response.json({
+            ok: false,
+            code: 'body',
+            why: `The stream gave more bytes than its length (${length}), so nothing was stored.`,
+          });
+        return Response.json({ ok: false, code: 'too-big', why: `A file may be ${FILES.fileBytes / 1048576} MB.` });
+      }
+      if (stream && length !== null && body.length !== length)
+        return Response.json({
+          ok: false,
+          code: 'body',
+          why: `The stream gave fewer bytes than its length (${length}), so nothing was stored.`,
+        });
+      const opts = { public: q.get('public') === '1', ...(q.get('type') ? { type: q.get('type') } : {}) };
+      return Response.json(await files.put(q.get('path') ?? '', body, opts));
+    }
+    if (op === 'files.get') {
+      let args = [];
+      try {
+        args = await request.json();
+      } catch {
+        /* no body */
+      }
+      const a = await files.get(String(args?.[0] ?? ''));
+      if (!a.ok) return Response.json(a);
+      const { body, ...facts } = a;
+      return new Response(body, { headers: { 'x-gf-answer': JSON.stringify(facts) } });
+    }
     let args = [];
     try {
       args = await request.json();
@@ -375,7 +443,38 @@ export function createPlatformStore({
     await writing;
     if (data) await write();
   }
-  return { slug, gameId, run, pageSubmit, handleHost, devToken, verifyPlayer, readToken, flush, file };
+  return {
+    slug,
+    gameId,
+    run,
+    pageSubmit,
+    handleHost,
+    devToken,
+    verifyPlayer,
+    readToken,
+    flush,
+    file,
+    files,
+    setFilesOrigin: o => {
+      filesOrigin = o;
+    },
+  };
+}
+
+/**
+ * A request's body, or null when it passes `cap` bytes. Past the cap the rest is read and dropped (up to a bound), so
+ * the request ends cleanly: an answer sent while workerd is still sending the body leaves its connection half open.
+ */
+async function readCapped(request, cap, drain = 64 * 1024 * 1024) {
+  if (!request.body) return Buffer.alloc(0);
+  const chunks = [];
+  let size = 0;
+  for await (const c of request.body) {
+    size += c.byteLength;
+    if (size > cap + drain) return null;
+    if (size <= cap) chunks.push(Buffer.from(c));
+  }
+  return size > cap ? null : Buffer.concat(chunks);
 }
 
 // ---- the page's side: /platform/v1/* on the dev host -------------------------------------------------------------

package/lib/dev/server.mjs

@@ -3,11 +3,14 @@
 //   env.GF        the platform stand-in (./gf-local.js → ./platform.mjs)
 //   env.<binding> each Durable Object gamesfun.json declares, with SQLite storage kept in <game>/.gamesfun/dev-server/
 //                 (so its state survives a restart, as hosted)
+//   env.DB        the game's database (gamesfun.json → database, under its own binding): a local D1 kept in
+//                 <game>/.gamesfun/dev/d1/, its migrations applied at each start and reload (./d1.mjs); a migration
+//                 changed after it was applied here keeps the server down, as xgames.fun refuses that publish
 //   env.GF_GAME_ID, GF_SLUG, GF_CONTRACT   as hosted (the game id here is dev-<folder>)
 //   env.<NAME>    the owner's secrets: hosted, set with `xcore secret set`; here, from the game folder's .dev.vars
 //                 (KEY=value lines; a dot file, so publish never sends it), read again at each reload
 // The dev host sends the page's /game/api/* here as /api/* (HTTP through dispatchFetch; WebSockets through a plain TCP
-// pipe to Miniflare's own port), with every x-gf-* header removed, as games.fun's dispatch removes them. What is not
+// pipe to Miniflare's own port), with every x-gf-* header removed, as xgames.fun's dispatch removes them. What is not
 // the same as hosted: no CPU or request limits, and outbound fetch is not limited to backend.json's origins.
 import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
@@ -20,10 +23,12 @@ import { bundleServer } from '../server/bundle.mjs';
 import { COMPATIBILITY_DATE, SERVER_FORMAT } from '../server/contract.mjs';
 import { loadMiniflare } from './tools.mjs';
 import { secretNameProblem } from '../server/secrets.mjs';
+import { applyMigrations, d1Options } from './d1.mjs';
+import { stateFolder } from '../state-folder.mjs';
 
 const GF_LOCAL = fileURLToPath(new URL('./gf-local.js', import.meta.url));
 
-/** Miniflare's options (v4) for one game's bundle: the bindings games.fun's upload gives it, the same runtime. */
+/** Miniflare's options (v4) for one game's bundle: the bindings xgames.fun's upload gives it, the same runtime. */
 export function workerOptions({
   code,
   objects,
@@ -33,11 +38,14 @@ export function workerOptions({
   gfSource,
   wasm = [],
   secrets = {},
+  database = null,
+  dir = null,
 }) {
   return {
     host: '127.0.0.1',
     port: 0,
     ...(persist ? { durableObjectsPersist: persist } : {}),
+    ...(database?.has ? { d1Persist: d1Options(dir, database.binding).d1Persist } : {}),
     workers: [
       {
         name: 'game',
@@ -48,6 +56,7 @@ export function workerOptions({
         compatibilityDate: COMPATIBILITY_DATE,
         compatibilityFlags: [...compatibilityFlags],
         durableObjects: Object.fromEntries(objects.map(o => [o.binding, { className: o.class, useSQLite: true }])),
+        ...(database?.has ? { d1Databases: d1Options(dir, database.binding).d1Databases } : {}),
         bindings: { ...secrets, GF_GAME_ID: store.gameId, GF_SLUG: store.slug, GF_CONTRACT: SERVER_FORMAT },
         serviceBindings: { GF: { name: 'gf-local', entrypoint: 'GameServices' } },
       },
@@ -67,7 +76,7 @@ const HOP = /^(host|connection|keep-alive|transfer-encoding|upgrade|x-gf-.*)$/i;
 export const DEV_VARS = '.dev.vars';
 
 /**
- * The game folder's .dev.vars as the server's secrets: {secrets: {NAME: value}, names, warnings}. A name games.fun would
+ * The game folder's .dev.vars as the server's secrets: {secrets: {NAME: value}, names, warnings}. A name xgames.fun would
  * refuse (lowercase, GF_*, a room's binding) is left out with a warning, so the code meets the same rules here.
  */
 export async function readDevVars(dir, objects = []) {
@@ -104,13 +113,17 @@ export async function startLocalServer({
 }) {
   const { Miniflare } = await loadMiniflare({ log, env });
   const gfSource = await readFile(GF_LOCAL, 'utf8');
+  await stateFolder(dir);
   let mf = null,
     url = null,
+    closed = false, // close() was called: no start after it makes a Miniflare again
     chain = Promise.resolve();
   let state = { running: false, error: null, problems: [], warnings: [], objects: [] };
 
   async function start() {
+    if (closed) return state;
     const b = await bundleServer(dir, { minify: false });
+    if (closed) return state;
     if (!b.has) {
       state = { running: false, error: 'this game has no server/', problems: [], warnings: [], objects: [] };
       return state;
@@ -123,7 +136,10 @@ export async function startLocalServer({
       ),
     ];
     const objects = b.objects.map(o => o.class);
-    const vars = await readDevVars(dir, b.objects);
+    const vars = await readDevVars(dir, [
+      ...b.objects,
+      ...(b.database?.has ? [{ class: 'the database', binding: b.database.binding }] : []),
+    ]);
     warnings.push(...vars.warnings);
     if (b.problems.length || !b.code) {
       if (mf) await mf.dispose().catch(() => {});
@@ -147,12 +163,44 @@ export async function startLocalServer({
       store,
       gfSource,
       secrets: vars.secrets,
+      database: b.database,
+      dir,
     });
     try {
       if (mf) await mf.setOptions(opts);
       else mf = new Miniflare(opts);
       url = await mf.ready;
-      state = { running: true, error: null, problems: [], warnings, objects, secrets: vars.names, inputs: b.inputs };
+      // the game's database: the migrations not applied here yet, in order; a refused one keeps the server down
+      let migrated = [];
+      if (b.database?.has) {
+        const m = await applyMigrations(dir, await mf.getD1Database(b.database.binding), b.database.migrations);
+        if (!m.ok) {
+          await mf.dispose().catch(() => {});
+          mf = url = null;
+          state = {
+            running: false,
+            error: "the game's database did not take its migrations",
+            problems: [`${m.why}.`],
+            warnings,
+            objects,
+            inputs: b.inputs,
+            migration: { code: m.code, name: m.name },
+          };
+          return state;
+        }
+        migrated = m.applied;
+      }
+      state = {
+        running: true,
+        error: null,
+        problems: [],
+        warnings,
+        objects,
+        secrets: vars.names,
+        inputs: b.inputs,
+        ...(b.database?.has ? { database: { binding: b.database.binding, migrated } } : {}),
+        ...(b.stock ? { stock: true } : {}),
+      };
     } catch (e) {
       if (mf) await mf.dispose().catch(() => {});
       mf = url = null;
@@ -242,6 +290,7 @@ export async function startLocalServer({
     },
     reload: () => queue(start),
     async close() {
+      closed = true;
       await chain;
       if (mf) await mf.dispose().catch(() => {});
       mf = null;

package/lib/dev/tools.mjs

@@ -3,8 +3,9 @@
 // server/, into the folder of this package's state (../config-dir.mjs → state/deps/server-tools-<miniflare version>).
 // GAMESFUN_SERVER_DEPS=<folder> uses one installed elsewhere (a folder whose node_modules has miniflare).
 import { spawnSync } from 'node:child_process';
-import { existsSync, mkdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
+import os from 'node:os';
 import path from 'node:path';
 import { userConfigDir } from '../config-dir.mjs';
 
@@ -35,30 +36,82 @@ function npmCli() {
 
 const pause = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 
-/** Install the tools when they are not there yet (one process at a time; whole or not at all). → the folder */
-export function ensureTools({ log = m => console.error(m), env = process.env } = {}) {
-  const dir = toolsFolder(env);
-  if (ready(dir)) return dir;
-  if (env.GAMESFUN_SERVER_DEPS) throw new Error(`GAMESFUN_SERVER_DEPS (${dir}) has no node_modules/miniflare`);
-  mkdirSync(dir, { recursive: true });
+/** Is process `pid` running (on this machine)? EPERM: it is, as another user's. */
+const alive = pid => {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return e.code === 'EPERM';
+  }
+};
+
+/**
+ * The install lock, <tools folder>/.installing, is in one of three states:
+ *   free   no folder: take it (mkdir, which only one process wins), and write `<pid> <host>` in its pid file.
+ *   held   the folder, by an xcore still installing: wait for it (up to waitMs), then say another xcore is installing.
+ *   stale  the folder, left by an install that was stopped (Ctrl-C kills xcore before it can remove it): taken over at
+ *          once. Stale when its pid file names a process on this machine that is gone, or (no pid file: an xcore before
+ *          this one wrote none; or a pid on another machine, or one reused since) when it is older than twice waitMs.
+ */
+function lockState(lock, waitMs) {
+  let age;
+  try {
+    age = Date.now() - statSync(lock).mtimeMs;
+  } catch {
+    return 'free';
+  }
+  let holder = null;
+  try {
+    const [pid, host] = readFileSync(path.join(lock, 'pid'), 'utf8').trim().split(/\s+/);
+    if (Number.isSafeInteger(Number(pid)) && Number(pid) > 0) holder = { pid: Number(pid), host };
+  } catch {
+    /* no pid file */
+  }
+  if (holder && holder.host === os.hostname() && !alive(holder.pid)) return 'stale';
+  return age > waitMs * 2 ? 'stale' : 'held';
+}
+
+/**
+ * Take the install lock in `dir` (one install at a time), waiting while another xcore holds it. → release(), or null
+ * when `done()` turned true while waiting (the other xcore installed the tools). Once held, any .stage-<pid> folder
+ * there is a stopped install's, and goes.
+ */
+export function takeInstallLock(dir, { waitMs = WAIT_MS, done = () => false } = {}) {
   const lock = path.join(dir, '.installing');
   const started = Date.now();
   for (;;) {
     try {
       mkdirSync(lock);
       break;
-    } catch {
-      if (ready(dir)) return dir;
-      let age = 0;
-      try {
-        age = Date.now() - statSync(lock).mtimeMs;
-      } catch {}
-      if (age > WAIT_MS * 2) rmSync(lock, { recursive: true, force: true });
-      else if (Date.now() - started > WAIT_MS)
-        throw new Error(`another xcore is still installing the server tools in ${dir}`);
-      else pause(500);
+    } catch (e) {
+      if (e.code !== 'EEXIST') throw e;
+      if (done()) return null;
+      const state = lockState(lock, waitMs);
+      if (state === 'stale') rmSync(lock, { recursive: true, force: true });
+      else if (state === 'held' && Date.now() - started > waitMs)
+        throw new Error(`another xcore is still installing the server tools in ${dir}`, { cause: e });
+      else if (state === 'held') pause(500);
     }
   }
+  try {
+    writeFileSync(path.join(lock, 'pid'), `${process.pid} ${os.hostname()}\n`);
+  } catch {
+    /* without it, the lock goes stale by its age */
+  }
+  for (const name of readdirSync(dir))
+    if (/^\.stage-\d+$/.test(name)) rmSync(path.join(dir, name), { recursive: true, force: true });
+  return () => rmSync(lock, { recursive: true, force: true });
+}
+
+/** Install the tools when they are not there yet (one process at a time; whole or not at all). → the folder */
+export function ensureTools({ log = m => console.error(m), env = process.env } = {}) {
+  const dir = toolsFolder(env);
+  if (ready(dir)) return dir;
+  if (env.GAMESFUN_SERVER_DEPS) throw new Error(`GAMESFUN_SERVER_DEPS (${dir}) has no node_modules/miniflare`);
+  mkdirSync(dir, { recursive: true });
+  const release = takeInstallLock(dir, { done: () => ready(dir) });
+  if (!release) return dir;
   try {
     if (ready(dir)) return dir;
     const cli = npmCli();
@@ -90,7 +143,7 @@ export function ensureTools({ log = m => console.error(m), env = process.env } =
     rmSync(stage, { recursive: true, force: true });
     return dir;
   } finally {
-    rmSync(lock, { recursive: true, force: true });
+    release();
   }
 }