@surph_ai/cli 0.0.33 → 0.0.34

package/board/deploy.js

@@ -0,0 +1,232 @@
+#!/usr/bin/env node
+// Deploy a board to Surph (`npm run deploy` → `surph-board-deploy`), run from
+// the board's folder. Built up step by step; so far:
+//
+//   1. Zip the board's site: every file in the folder except the project
+//      files (package.json, package-lock.json, README.md, board.json,
+//      node_modules), dot-files (.env, .git, .DS_Store) and the zip itself.
+//      Written to site.zip in the board's folder.
+//   2. Ask Surph for an upload address for it: boards/<slug>/site/site.zip on
+//      S3, the slug from board.json.
+//   3. Upload site.zip there, then delete the local site.zip (it's kept when
+//      the upload fails, to look at).
+//   4. Unzip it on S3 into boards/<slug>/site/ (the board's pages and assets).
+//   5. Upload board.json to boards/<slug>/board.json — last, so the live
+//      board's definition only changes once its site is up.
+//
+// Node's built-ins only (zlib for compression, fetch for the upload; Node 18+).
+
+const fs = require('fs');
+const path = require('path');
+const zlib = require('zlib');
+
+const ROOT = process.cwd();
+const ZIP_NAME = 'site.zip';
+const RPC_URL = 'https://rpc.surph.ai';
+// Unzips an uploaded archive in place on S3 (the `sync` action).
+const SYNC_URL = 'https://e3lunom2hka3u6iiotrlzrcez40uwdnk.lambda-url.us-east-1.on.aws/';
+const SLUG_RE = /^[a-z0-9][a-z0-9-]*$/;
+
+// Left out at the board's top level: the project's own files.
+const SKIP_AT_ROOT = new Set(['package.json', 'package-lock.json', 'README.md', 'board.json', ZIP_NAME]);
+
+// The files to deploy, as paths relative to the board's folder ('/'-separated).
+const siteFiles = (dir = ROOT, rel = '') => {
+  const out = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const name = entry.name;
+    if (name.startsWith('.') || name === 'node_modules') continue;
+    if (!rel && SKIP_AT_ROOT.has(name)) continue;
+    const relPath = rel ? `${rel}/${name}` : name;
+    const full = path.join(dir, name);
+    if (entry.isDirectory()) out.push(...siteFiles(full, relPath));
+    else if (entry.isFile()) out.push(relPath);
+  }
+  return out.sort();
+};
+
+// CRC-32 (the zip format's checksum).
+const CRC_TABLE = (() => {
+  const table = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    table[n] = c >>> 0;
+  }
+  return table;
+})();
+const crc32 = (buf) => {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+};
+
+// A file's time in the zip format's (MS-DOS) date and time fields.
+const dosTime = (date) => ({
+  time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2),
+  date: ((date.getFullYear() - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
+});
+
+// A zip archive of the given files (deflated, or stored when that's smaller).
+const zipFiles = (files) => {
+  const parts = [];
+  const central = [];
+  let offset = 0;
+  for (const rel of files) {
+    const full = path.join(ROOT, ...rel.split('/'));
+    const data = fs.readFileSync(full);
+    const deflated = zlib.deflateRawSync(data, { level: 9 });
+    const stored = deflated.length >= data.length;
+    const body = stored ? data : deflated;
+    const method = stored ? 0 : 8;
+    const crc = crc32(data);
+    const { time, date } = dosTime(fs.statSync(full).mtime);
+    const name = Buffer.from(rel, 'utf8');
+
+    const local = Buffer.alloc(30);
+    local.writeUInt32LE(0x04034b50, 0);
+    local.writeUInt16LE(20, 4); // version needed
+    local.writeUInt16LE(0x0800, 6); // UTF-8 names
+    local.writeUInt16LE(method, 8);
+    local.writeUInt16LE(time, 10);
+    local.writeUInt16LE(date, 12);
+    local.writeUInt32LE(crc, 14);
+    local.writeUInt32LE(body.length, 18);
+    local.writeUInt32LE(data.length, 22);
+    local.writeUInt16LE(name.length, 26);
+    local.writeUInt16LE(0, 28);
+    parts.push(local, name, body);
+
+    const entry = Buffer.alloc(46);
+    entry.writeUInt32LE(0x02014b50, 0);
+    entry.writeUInt16LE(20, 4); // version made by
+    entry.writeUInt16LE(20, 6); // version needed
+    entry.writeUInt16LE(0x0800, 8);
+    entry.writeUInt16LE(method, 10);
+    entry.writeUInt16LE(time, 12);
+    entry.writeUInt16LE(date, 14);
+    entry.writeUInt32LE(crc, 16);
+    entry.writeUInt32LE(body.length, 20);
+    entry.writeUInt32LE(data.length, 24);
+    entry.writeUInt16LE(name.length, 28);
+    entry.writeUInt32LE(offset, 42);
+    central.push(entry, name);
+
+    offset += local.length + name.length + body.length;
+  }
+  const centralBuf = Buffer.concat(central);
+  const end = Buffer.alloc(22);
+  end.writeUInt32LE(0x06054b50, 0);
+  end.writeUInt16LE(files.length, 8);
+  end.writeUInt16LE(files.length, 10);
+  end.writeUInt32LE(centralBuf.length, 12);
+  end.writeUInt32LE(offset, 16);
+  return Buffer.concat([...parts, centralBuf, end]);
+};
+
+const fail = (message) => {
+  console.error(message);
+  process.exit(1);
+};
+
+// The board's slug, from its board.json.
+const boardSlug = () => {
+  let board;
+  try {
+    board = JSON.parse(fs.readFileSync(path.join(ROOT, 'board.json'), 'utf8'));
+  } catch (error) {
+    fail(`board.json isn't valid JSON: ${error.message}`);
+  }
+  const slug = typeof board?.slug === 'string' ? board.slug.trim() : '';
+  if (!SLUG_RE.test(slug)) fail('board.json needs a "slug": lowercase letters, digits and dashes (e.g. "my-board").');
+  return slug;
+};
+
+// A presigned S3 address to PUT a file to, under boards/<slug>/ (good for an hour).
+const uploadUrlFor = async (slug, filename, filetype) => {
+  const res = await fetch(`${RPC_URL}/s3/upload_url`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', Accept: 'application/json', 'x-surph-client': 'surph-sdk' },
+    body: JSON.stringify({
+      folder: slug,
+      filename,
+      filetype,
+      bucket: 'surph',
+      toStatic: 'false',
+      collection: 'boards',
+    }),
+  });
+  const json = await res.json().catch(() => null);
+  const uploadUrl = json?.response?.uploadUrl;
+  if (!res.ok || !uploadUrl) fail(`Couldn't get an upload address for ${filename}: ${json?.response?.error || `HTTP ${res.status}`}`);
+  return uploadUrl;
+};
+
+// PUT a file to its presigned address. S3 answers 200; anything else is an
+// XML error, whose code and message say why.
+const upload = async (uploadUrl, body, filetype, filename) => {
+  const res = await fetch(uploadUrl, {
+    method: 'PUT',
+    headers: { 'Content-Type': filetype },
+    body,
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    const code = (text.match(/<Code>([^<]+)<\/Code>/) || [])[1] || `HTTP ${res.status}`;
+    const message = (text.match(/<Message>([^<]+)<\/Message>/) || [])[1] || '';
+    fail(`Upload of ${filename} failed: ${code}${message ? ` — ${message}` : ''}`);
+  }
+};
+
+// Unzip boards/<slug>/site/site.zip on S3 into boards/<slug>/site/.
+const unzipOnS3 = async (slug) => {
+  const res = await fetch(SYNC_URL, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', Accept: 'application/json', 'Turbo-Client': 'portal' },
+    body: JSON.stringify({
+      action: 'sync',
+      args: { project: slug, file: ZIP_NAME, bucket: 'surph', folder: 'site' },
+    }),
+  });
+  const text = await res.text().catch(() => '');
+  let json = null;
+  try { json = JSON.parse(text); } catch { /* not JSON */ }
+  const error = json?.error || json?.response?.error || json?.data?.error;
+  if (!res.ok || error) {
+    fail(`Unzipping ${ZIP_NAME} on S3 failed: ${typeof error === 'string' ? error : (error?.message || text.slice(0, 300) || `HTTP ${res.status}`)}`);
+  }
+};
+
+const main = async () => {
+  if (!fs.existsSync(path.join(ROOT, 'board.json'))) {
+    fail(`No board.json in ${ROOT}. Run this from the board's folder.`);
+  }
+  const slug = boardSlug();
+  const files = siteFiles();
+  if (!files.length) fail('Nothing to deploy: the board has no site files.');
+
+  const zip = zipFiles(files);
+  fs.writeFileSync(path.join(ROOT, ZIP_NAME), zip);
+  console.log(`Zipped ${files.length} file${files.length === 1 ? '' : 's'} into ${ZIP_NAME} (${(zip.length / 1024).toFixed(1)} KB):`);
+  for (const file of files) console.log(`  ${file}`);
+
+  console.log(`\nDeploying the board "${slug}"…`);
+  const zipUrl = await uploadUrlFor(slug, `site/${ZIP_NAME}`, 'application/zip');
+  await upload(zipUrl, zip, 'application/zip', ZIP_NAME);
+  console.log(`Uploaded: ${zipUrl.split('?')[0]}`);
+  fs.rmSync(path.join(ROOT, ZIP_NAME), { force: true });
+
+  await unzipOnS3(slug);
+  console.log(`Unzipped into: https://surph.s3.us-east-1.amazonaws.com/boards/${slug}/site/`);
+
+  const boardUrl = await uploadUrlFor(slug, 'board.json', 'application/json');
+  await upload(boardUrl, fs.readFileSync(path.join(ROOT, 'board.json')), 'application/json', 'board.json');
+  console.log(`Uploaded: ${boardUrl.split('?')[0]}`);
+};
+
+const run = () => main().catch((error) => fail(`Deploy failed: ${error.message}`));
+
+// `surph board-deploy` calls run(); running this file directly runs it too.
+if (require.main === module) run();
+
+module.exports = { run };

package/board/server.js

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+// The board's local dev server (`npm run dev` → `surph-board-dev`): serves
+// the board folder it's run from (the current directory) at
+// http://localhost:4180 for Surph's dev mode (Settings → Developer → Run
+// Surph as a local board). Node's built-ins only.
+//
+//   GET  /<file>             This folder's files, never cached, with CORS so
+//                            Surph (another address) can read board.json.
+//   GET  /__surph/assets     The image files in assets/ (the logo picker).
+//   PUT  /board.json         Surph's Local board panel saving the board. Only
+//                            from Surph's own addresses, only this one file,
+//                            only a JSON object with a config.pages list.
+//
+// It listens on this machine only (127.0.0.1 and ::1), never the network.
+
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+
+const PORT = Number(process.env.PORT) || 4180;
+// The board's folder: where `npm run dev` runs it, not this package's folder
+// (it's installed under node_modules/@surph_ai/sdk).
+const ROOT = process.cwd();
+const MAX_BOARD_BYTES = 256 * 1024;
+
+// The addresses whose pages may save board.json.
+const SAVE_ORIGINS = [
+  /^https:\/\/([a-z0-9-]+\.)?surph\.ai$/,
+  /^http:\/\/localhost:\d+$/,
+  /^http:\/\/127\.0\.0\.1:\d+$/,
+];
+const canSave = (origin) => typeof origin === 'string' && SAVE_ORIGINS.some((re) => re.test(origin));
+
+const TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.svg': 'image/svg+xml',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+  '.ico': 'image/x-icon',
+  '.txt': 'text/plain; charset=utf-8',
+  '.md': 'text/markdown; charset=utf-8',
+};
+const IMAGE = /\.(png|jpe?g|gif|webp|svg)$/i;
+
+const send = (res, status, body, headers = {}) => {
+  res.writeHead(status, { 'Cache-Control': 'no-store', ...headers });
+  res.end(body);
+};
+
+const cors = (req) => {
+  const origin = req.headers.origin;
+  const headers = { 'Access-Control-Allow-Origin': '*' };
+  // A save is only allowed from Surph; the browser asks first (preflight).
+  // Chrome asks before a public site (surph.ai) reaches a local address.
+  if (req.headers['access-control-request-private-network'] === 'true') {
+    headers['Access-Control-Allow-Private-Network'] = 'true';
+  }
+  if (canSave(origin)) {
+    headers['Access-Control-Allow-Origin'] = origin;
+    headers['Access-Control-Allow-Methods'] = 'GET, PUT, OPTIONS';
+    headers['Access-Control-Allow-Headers'] = 'Content-Type';
+    headers.Vary = 'Origin';
+  }
+  return headers;
+};
+
+// A path inside this folder, or null (no way out of it, no dot-files).
+const inside = (urlPath) => {
+  let rel;
+  try { rel = decodeURIComponent(urlPath.split('?')[0]); } catch { return null; }
+  if (rel.endsWith('/')) rel += 'index.html';
+  const full = path.normalize(path.join(ROOT, rel));
+  if (full !== ROOT && !full.startsWith(ROOT + path.sep)) return null;
+  if (path.relative(ROOT, full).split(path.sep).some((part) => part.startsWith('.') || part === 'node_modules')) return null;
+  return full;
+};
+
+const readBody = (req) => new Promise((resolve, reject) => {
+  let size = 0;
+  const chunks = [];
+  req.on('data', (chunk) => {
+    size += chunk.length;
+    if (size > MAX_BOARD_BYTES) {
+      reject(new Error('board.json is too large'));
+      req.destroy();
+      return;
+    }
+    chunks.push(chunk);
+  });
+  req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+  req.on('error', reject);
+});
+
+const saveBoard = async (req, res, headers) => {
+  if (!canSave(req.headers.origin)) {
+    send(res, 403, JSON.stringify({ error: 'board.json can only be saved from Surph.' }), { ...headers, 'Content-Type': 'application/json' });
+    return;
+  }
+  let board;
+  try {
+    board = JSON.parse(await readBody(req));
+  } catch (error) {
+    send(res, 400, JSON.stringify({ error: error.message || 'Not JSON.' }), { ...headers, 'Content-Type': 'application/json' });
+    return;
+  }
+  if (!board || typeof board !== 'object' || Array.isArray(board) || !Array.isArray(board.config && board.config.pages)) {
+    send(res, 400, JSON.stringify({ error: 'board.json must be an object with a config.pages list.' }), { ...headers, 'Content-Type': 'application/json' });
+    return;
+  }
+  // Written via a temporary file, so a failed write never leaves half a file.
+  const file = path.join(ROOT, 'board.json');
+  const tmp = `${file}.saving`;
+  fs.writeFileSync(tmp, `${JSON.stringify(board, null, 2)}\n`);
+  fs.renameSync(tmp, file);
+  console.log(`Saved board.json (${new Date().toLocaleTimeString()})`);
+  send(res, 200, JSON.stringify({ ok: true }), { ...headers, 'Content-Type': 'application/json' });
+};
+
+const handle = async (req, res) => {
+  const headers = cors(req);
+  const urlPath = req.url || '/';
+  try {
+    if (req.method === 'OPTIONS') {
+      send(res, 204, '', headers);
+      return;
+    }
+    if (req.method === 'PUT' && urlPath.split('?')[0] === '/board.json') {
+      await saveBoard(req, res, headers);
+      return;
+    }
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+      send(res, 405, 'Method not allowed', headers);
+      return;
+    }
+    if (urlPath.split('?')[0] === '/__surph/assets') {
+      const dir = path.join(ROOT, 'assets');
+      const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((name) => IMAGE.test(name)).sort() : [];
+      send(res, 200, JSON.stringify({ images: files }), { ...headers, 'Content-Type': 'application/json' });
+      return;
+    }
+    const full = inside(urlPath);
+    if (!full || !fs.existsSync(full) || !fs.statSync(full).isFile()) {
+      send(res, 404, 'Not found', headers);
+      return;
+    }
+    const type = TYPES[path.extname(full).toLowerCase()] || 'application/octet-stream';
+    send(res, 200, req.method === 'HEAD' ? '' : fs.readFileSync(full), { ...headers, 'Content-Type': type });
+  } catch (error) {
+    send(res, 500, String(error && error.message || error), headers);
+  }
+};
+
+// Start serving the current folder. localhost may be reached over IPv4 or
+// IPv6: listen on both loopbacks.
+const start = () => {
+  http.createServer(handle)
+    .on('error', (error) => {
+      console.error(error.code === 'EADDRINUSE'
+        ? `Port ${PORT} is already in use: another board server may be running. Stop it, or set PORT to use another port.`
+        : `Board dev server failed: ${error.message}`);
+      process.exit(1);
+    })
+    .listen(PORT, '127.0.0.1', () => {
+      console.log(`Board dev server: http://localhost:${PORT} (serving ${ROOT})`);
+      console.log('In Surph: Settings → Developer → Run Surph as a local board.');
+    });
+  http.createServer(handle).listen(PORT, '::1').on('error', () => { /* no IPv6 loopback */ });
+};
+
+// `surph board-server` calls start(); running this file directly starts it too.
+if (require.main === module) start();
+
+module.exports = { start };

package/cli.js

@@ -20,6 +20,8 @@ const AdmZip = require('adm-zip')
 // const auth = require('./auth')
 const deploy = require('./deploy')
 const scaffold = require('./scaffold')
+const boardDeploy = require('./board/deploy')
+const boardServer = require('./board/server')
 const { ensureChat, chatNodePath } = require('./toolserver/ensure-chat')
 const utils = require('./utils')
 const { openBrowser, openBrowserWhenReady } = require('./utils/browser')
@@ -297,6 +299,32 @@ program.command('board <name>')
 .description('create new Surph board')
 .action(name => scaffold(name, 'board'))
 
+program.command('board-deploy')
+.description('Deploy board to Surph')
+.action(async (cmd, options) => {
+  try {
+    // From the board's folder: zip its site, upload and unzip it, then
+    // upload board.json (board/deploy.js).
+    await boardDeploy.run()
+  } catch (error) {
+    utils.printError(error)
+    program.error(`Unexpected error: ${error}`, ERRORS.UNSPECIFIED)
+  }
+})
+
+program.command('board-server')
+.description('local board dev server')
+.action(async (cmd, options) => {
+  try {
+    // Serves the board's folder (the current directory) at
+    // http://localhost:4180 for Surph's dev mode (board/server.js).
+    boardServer.start()
+  } catch (error) {
+    utils.printError(error)
+    program.error(`Unexpected error: ${error}`, ERRORS.UNSPECIFIED)
+  }
+})
+
 program.command('manifest')
 .description('update manifest for Surph tool')
 .action(async (cmd, options) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@surph_ai/cli",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "description": "",
   "main": "index.js",
   "types": "./types.d.ts",

package/scaffold/board/README.md

@@ -400,7 +400,7 @@ A field marked "omitted" is missing from the object, not `null`. Every URL is `h
 
 ## Developing locally
 
-1. Install once with `npm install`, then run `npm run dev`. It starts the board dev server from `@surph_ai/sdk` (`surph-board-dev`), which serves this folder at `http://localhost:4180` with caching off, lets Surph read it from another address, and lets Surph's Local board panel save `board.json`. It only listens on your own machine.
+1. Run `npm run dev` (nothing to install). It starts the board dev server from the Surph CLI (`npx @surph_ai/cli board-server`), which serves this folder at `http://localhost:4180` with caching off, lets Surph read it from another address, and lets Surph's Local board panel save `board.json`. It only listens on your own machine.
 2. Keep it running while you work; the terminal shows a line each time Surph saves `board.json`.
 3. Open `http://localhost:4180/` to check a page loads without console errors. It shows its empty layout there (`hosted` is false); real data only appears inside the Surph canvas.
 4. **Run Surph as your board with dev mode.** Keep `npm run dev` running. In Surph, open **Settings → Developer**, turn on **Run Surph as a local board**, and keep the address `http://localhost:4180`. Surph reloads as the board `board.json` describes: its name and logo in the header, its contexts and instructions in the chat, and its pages in the canvas (it opens on your start page, the Pages menu lists your pages, and the chat's model sees their `shows` lines). The status under the address says what loaded, or why it couldn't read `board.json`; until it loads, Surph runs as usual. A **DEV** label in the canvas header shows your board is loaded. While it's on, anything the chat saves goes to your own space in Surph, not to a board's shared space.
@@ -410,7 +410,7 @@ A field marked "omitted" is missing from the object, not `null`. Every URL is `h
 
 ## Deploying
 
-`npm run deploy` (`surph-board-deploy` from `@surph_ai/sdk`) publishes the board to Surph. Run it from the board's folder. It needs Node 18 or later and nothing else installed. In order, it:
+`npm run deploy` (`npx @surph_ai/cli board-deploy`, from the Surph CLI) publishes the board to Surph. Run it from the board's folder. It needs Node 18 or later and nothing else installed. In order, it:
 
 1. **Reads the board's `slug`** from `board.json` (lowercase letters, digits and dashes). The slug names the board's place in Surph; without a valid one it stops before doing anything.
 2. **Zips the site** into `site.zip`: every file in the folder except `package.json`, `package-lock.json`, `README.md`, `board.json`, `node_modules`, dot-files (`.env`, `.git`, `.DS_Store`) and `site.zip` itself. It lists the files it zipped.

package/scaffold/board/package.json

@@ -3,14 +3,11 @@
   "version": "1.0.0",
   "description": "",
   "scripts": {
-    "dev": "surph-board-dev",
-    "deploy": "surph-board-deploy"
+    "dev": "npx @surph_ai/cli board-server",
+    "deploy": "npx @surph_ai/cli board-deploy"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
-  "type": "commonjs",
-  "dependencies": {
-    "@surph_ai/sdk": "latest"
-  }
+  "type": "commonjs"
 }