@peerbit/server 1.1.2 → 3.0.0

package/src/server.ts

@@ -1,5 +1,5 @@
 import http from "http";
-import { fromBase64 } from "@peerbit/crypto";
+import { Ed25519Keypair, fromBase64, sha256Base64Sync } from "@peerbit/crypto";
 import { deserialize } from "@dao-xyz/borsh";
 import {
 	Program,
@@ -10,90 +10,130 @@ import {
 import { waitFor } from "@peerbit/time";
 import { v4 as uuid } from "uuid";
 import {
-	checkExistPath,
 	getHomeConfigDir,
-	getCredentialsPath,
-	getPackageName,
-	loadPassword,
-	NotFoundError,
+	getNodePath,
+	getKeypair,
+	getTrustPath,
 } from "./config.js";
 import { setMaxListeners } from "events";
 import { create } from "./peerbit.js";
 import { Peerbit } from "peerbit";
 import { getSchema } from "@dao-xyz/borsh";
-import { StartByBase64, StartByVariant, StartProgram } from "./types.js";
+import {
+	InstallDependency,
+	StartByBase64,
+	StartByVariant,
+	StartProgram,
+} from "./types.js";
 import {
 	ADDRESS_PATH,
 	BOOTSTRAP_PATH,
+	TERMINATE_PATH,
 	INSTALL_PATH,
 	LOCAL_PORT,
 	PEER_ID_PATH,
 	PROGRAMS_PATH,
 	PROGRAM_PATH,
+	RESTART_PATH,
+	TRUST_PATH,
 } from "./routes.js";
-import { client } from "./client.js";
-
+import { Session } from "./session.js";
+import fs from "fs";
+import { exit } from "process";
+import { spawn, fork, execSync } from "child_process";
+import tmp from "tmp";
+import path from "path";
+import { base58btc } from "multiformats/bases/base58";
 import { dirname } from "path";
 import { fileURLToPath } from "url";
+import { Level } from "level";
+import { MemoryLevel } from "memory-level";
+import { Trust } from "./trust.js";
+import { getBody, verifyRequest } from "./signes-request.js";
+import { cli } from "./cli.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-export const createPassword = async (): Promise<string> => {
-	const fs = await import("fs");
-	const configDir = await getHomeConfigDir();
-	const credentialsPath = await getCredentialsPath(configDir);
-	if (await checkExistPath(credentialsPath)) {
-		throw new Error(
-			"Config path for credentials: " + credentialsPath + ", already exist"
-		);
-	}
-	console.log(`Creating config folder ${configDir}`);
-
-	fs.mkdirSync(configDir, { recursive: true });
-	await waitFor(() => fs.existsSync(configDir));
-
-	console.log(`Created config folder ${configDir}`);
-
-	const password = uuid();
-	fs.writeFileSync(
-		credentialsPath,
-		JSON.stringify({ username: "admin", password })
-	);
-	console.log(`Created credentials at ${credentialsPath}`);
-	return password;
+export const stopAndWait = (server: http.Server) => {
+	let closed = false;
+	server.on("close", () => {
+		closed = true;
+	});
+	server.close();
+	return waitFor(() => closed);
 };
 
-export const loadOrCreatePassword = async (): Promise<string> => {
-	try {
-		return await loadPassword();
-	} catch (error) {
-		if (error instanceof NotFoundError) {
-			return createPassword();
-		}
-		throw error;
-	}
-};
 export const startServerWithNode = async (properties: {
-	directory?: string;
+	directory: string;
 	domain?: string;
 	bootstrap?: boolean;
+	newSession?: boolean;
+	ports?: {
+		node: number;
+		api: number;
+	};
+	restart?: () => void;
 }) => {
+	if (!fs.existsSync(properties.directory)) {
+		fs.mkdirSync(properties.directory, { recursive: true });
+	}
+	const keypair = await getKeypair(properties.directory);
 	const peer = await create({
-		directory: properties.directory,
+		directory:
+			properties.directory != null
+				? getNodePath(properties.directory)
+				: undefined,
 		domain: properties.domain,
+		listenPort: properties.ports?.node,
+		peerId: await keypair.toPeerId(),
 	});
 
 	if (properties.bootstrap) {
 		await peer.bootstrap();
 	}
+	const sessionDirectory =
+		properties.directory != null
+			? path.join(properties.directory, "session")
+			: undefined;
+
+	const session = new Session(
+		sessionDirectory
+			? new Level<string, Uint8Array>(sessionDirectory, {
+					valueEncoding: "view",
+					keyEncoding: "utf-8",
+			  })
+			: new MemoryLevel({ valueEncoding: "view", keyEncoding: "utf-8" })
+	);
+	if (!properties.newSession) {
+		for (const [string] of await session.imports.all()) {
+			await import(string);
+		}
+		for (const [address] of await session.programs.all()) {
+			// TODO args
+			try {
+				await peer.open(address, { timeout: 3000 });
+			} catch (error) {
+				console.error(error);
+			}
+		}
+	} else {
+		await session.clear();
+	}
 
-	const server = await startServer(peer);
+	const server = await startApiServer(peer, {
+		port: properties.ports?.api,
+		configDirectory:
+			properties.directory != null
+				? path.join(properties.directory, "server")
+				: undefined || getHomeConfigDir(),
+		session,
+	});
 	const printNodeInfo = async () => {
 		console.log("Starting node with address(es): ");
-		const id = await (await client()).peer.id.get();
+		const id = peer.peerId.toString();
 		console.log("id: " + id);
 		console.log("Addresses: ");
-		for (const a of await (await client()).peer.addresses.get()) {
+		for (const a of peer.getMultiaddrs()) {
 			console.log(a.toString());
 		}
 	};
@@ -101,27 +141,33 @@ export const startServerWithNode = async (properties: {
 	await printNodeInfo();
 	const shutDownHook = async (
 		controller: { stop: () => any },
-		server: {
-			close: () => void;
-		}
+		server: http.Server
 	) => {
-		const { exit } = await import("process");
-		process.on("SIGINT", async () => {
-			console.log("Shutting down node");
-			await server.close();
-			await controller.stop();
-			exit();
+		["SIGTERM", "SIGINT", "SIGUSR1", "SIGUSR2"].forEach((code) => {
+			process.on(code, async () => {
+				if (server.listening) {
+					console.log("Shutting down node");
+					await stopAndWait(server);
+					await waitFor(() => closed);
+					await controller.stop();
+				}
+				exit();
+			});
+		});
+		process.on("exit", async () => {
+			if (server.listening) {
+				console.log("Shutting down node");
+				await stopAndWait(server);
+				await waitFor(() => closed);
+				await controller.stop();
+			}
 		});
 	};
 	await shutDownHook(peer, server);
 	return { server, node: peer };
 };
 
-const getProgramFromPath = (
-	client: Peerbit,
-	req: http.IncomingMessage,
-	pathIndex: number
-): Program | undefined => {
+const getPathValue = (req: http.IncomingMessage, pathIndex: number): string => {
 	if (!req.url) {
 		throw new Error("Missing url");
 	}
@@ -133,42 +179,90 @@ const getProgramFromPath = (
 		throw new Error("Invalid path");
 	}
 	const address = decodeURIComponent(path[pathIndex]);
-	return client.handler.items.get(address);
+	return address;
 };
+function findPeerbitProgramFolder(inputDirectory: string): string | null {
+	let currentDir = path.resolve(inputDirectory);
+
+	while (currentDir !== "/") {
+		// Stop at the root directory
+		const nodeModulesPath = path.join(currentDir, "node_modules");
+		const packageJsonPath = path.join(
+			nodeModulesPath,
+			"@peerbit",
+			"program",
+			"package.json"
+		);
+
+		if (fs.existsSync(packageJsonPath)) {
+			const packageJsonContent = fs.readFileSync(packageJsonPath, "utf-8");
+			const packageData = JSON.parse(packageJsonContent);
+
+			if (packageData.name === "@peerbit/program") {
+				return currentDir;
+			}
+		}
+
+		currentDir = path.dirname(currentDir);
+	}
+
+	return null;
+}
 
-export const startServer = async (
+export const startApiServer = async (
 	client: ProgramClient,
-	port: number = LOCAL_PORT
+	properties: {
+		configDirectory: string;
+		session?: Session;
+		port?: number;
+	}
 ): Promise<http.Server> => {
-	const password = await loadOrCreatePassword();
+	const port = properties?.port ?? LOCAL_PORT;
+	if (!fs.existsSync(properties.configDirectory)) {
+		fs.mkdirSync(properties.configDirectory, { recursive: true });
+	}
 
-	const adminACL = (req: http.IncomingMessage): boolean => {
-		const auth = req.headers["authorization"];
-		if (!auth?.startsWith("Basic ")) {
-			return false;
-		}
-		const credentials = auth?.substring("Basic ".length);
-		const username = credentials.split(":")[0];
-		if (username !== "admin") {
-			return false;
-		}
-		if (password !== credentials.substring(username.length + 1)) {
-			return false;
-		}
-		return true;
+	const trust = new Trust(getTrustPath(properties.configDirectory));
+
+	const restart = async () => {
+		await client.stop();
+		await stopAndWait(server);
+
+		// We filter out the reset command, since restarting means that we want to resume something
+		spawn(
+			process.argv.shift()!,
+			[
+				...process.execArgv,
+				...process.argv.filter((x) => x !== "--reset" && x !== "-r"),
+			],
+			{
+				cwd: process.cwd(),
+				detached: true,
+				stdio: "inherit",
+				gid: process.getgid!(),
+			}
+		);
+		process.exit(0);
 	};
+	if (!client.peerId.equals(await client.identity.publicKey.toPeerId())) {
+		throw new Error("Expecting node identity to equal peerId");
+	}
 
-	const getBody = (
-		req: http.IncomingMessage,
-		callback: (body: string) => Promise<void> | void
-	) => {
-		let body = "";
-		req.on("data", function (d) {
-			body += d;
-		});
-		req.on("end", function () {
-			callback(body);
-		});
+	const getVerifiedBody = async (req: http.IncomingMessage) => {
+		const body = await getBody(req);
+		const result = await verifyRequest(
+			req.headers,
+			req.method!,
+			req.url!,
+			body
+		);
+		if (result.equals(client.identity.publicKey)) {
+			return body;
+		}
+		if (trust.isTrusted(result.hashcode())) {
+			return body;
+		}
+		throw new Error("Not trusted");
 	};
 
 	const e404 = "404";
@@ -186,15 +280,20 @@ export const startServer = async (
 
 			try {
 				if (req.url) {
-					if (
-						!req.url.startsWith(PEER_ID_PATH) &&
-						!req.url.startsWith(ADDRESS_PATH) &&
-						!(await adminACL(req))
-					) {
+					let body: string;
+					try {
+						body =
+							req.url.startsWith(PEER_ID_PATH) ||
+							req.url.startsWith(ADDRESS_PATH)
+								? await getBody(req)
+								: await getVerifiedBody(req);
+					} catch (error: any) {
 						res.writeHead(401);
-						res.end("Not authorized");
+						res.end("Not authorized: " + error.toString());
 						return;
-					} else if (req.url.startsWith(PROGRAMS_PATH)) {
+					}
+
+					if (req.url.startsWith(PROGRAMS_PATH)) {
 						if (client instanceof Peerbit === false) {
 							res.writeHead(400);
 							res.end("Server node is not running a native client");
@@ -203,9 +302,8 @@ export const startServer = async (
 						switch (req.method) {
 							case "GET":
 								try {
-									const keys = JSON.stringify([
-										...(client as Peerbit).handler.items.keys(),
-									]);
+									const ref = (client as Peerbit).handler?.items?.keys() || [];
+									const keys = JSON.stringify([...ref]);
 									res.setHeader("Content-Type", "application/json");
 									res.writeHead(200);
 									res.end(keys);
@@ -228,7 +326,9 @@ export const startServer = async (
 						switch (req.method) {
 							case "HEAD":
 								try {
-									const program = getProgramFromPath(client as Peerbit, req, 1);
+									const program = (client as Peerbit).handler?.items.get(
+										getPathValue(req, 1)
+									);
 									if (program) {
 										res.writeHead(200);
 										res.end();
@@ -247,13 +347,22 @@ export const startServer = async (
 									const url = new URL(req.url, "http://localhost:" + 1234);
 									const queryData = url.searchParams.get("delete");
 
-									const program = getProgramFromPath(client as Peerbit, req, 1);
+									const program = (client as Peerbit).handler?.items.get(
+										getPathValue(req, 1)
+									);
 									if (program) {
+										let closed = false;
 										if (queryData === "true") {
-											await program.drop();
+											closed = await program.drop();
 										} else {
-											await program.close();
+											closed = await program.close();
+										}
+										if (closed) {
+											await properties?.session?.programs.remove(
+												program.address
+											);
 										}
+
 										res.writeHead(200);
 										res.end();
 									} else {
@@ -266,45 +375,54 @@ export const startServer = async (
 								}
 								break;
 							case "PUT":
-								getBody(req, (body) => {
-									try {
-										const startArguments: StartProgram = JSON.parse(body);
+								try {
+									const startArguments: StartProgram = JSON.parse(body);
 
-										let program: Program;
-										if ((startArguments as StartByVariant).variant) {
-											const P = getProgramFromVariant(
-												(startArguments as StartByVariant).variant
-											);
-											if (!P) {
-												res.writeHead(400);
-												res.end(
-													"Missing program with variant: " +
-														(startArguments as StartByVariant).variant
-												);
-												return;
-											}
-											program = new P();
-										} else {
-											program = deserialize(
-												fromBase64((startArguments as StartByBase64).base64),
-												Program
+									let program: Program;
+									if ((startArguments as StartByVariant).variant) {
+										const P = getProgramFromVariant(
+											(startArguments as StartByVariant).variant
+										);
+										if (!P) {
+											res.writeHead(400);
+											res.end(
+												"Missing program with variant: " +
+													(startArguments as StartByVariant).variant
 											);
+											return;
 										}
-										client
-											.open(program) // TODO all users to pass args
-											.then((program) => {
-												res.writeHead(200);
-												res.end(program.address.toString());
-											})
-											.catch((error) => {
-												res.writeHead(400);
-												res.end("Failed to open program: " + error.toString());
-											});
-									} catch (error: any) {
-										res.writeHead(400);
-										res.end(error.toString());
+										program = new P();
+									} else {
+										program = deserialize(
+											fromBase64((startArguments as StartByBase64).base64),
+											Program
+										);
 									}
-								});
+									client
+										.open(program) // TODO all users to pass args
+										.then(async (program) => {
+											// TODO what if this is a reopen?
+											console.log(
+												"OPEN ADDRESS",
+												program.address,
+												(client as Peerbit).directory,
+												await client.services.blocks.has(program.address)
+											);
+											await properties?.session?.programs.add(
+												program.address,
+												new Uint8Array()
+											);
+											res.writeHead(200);
+											res.end(program.address.toString());
+										})
+										.catch((error) => {
+											res.writeHead(400);
+											res.end("Failed to open program: " + error.toString());
+										});
+								} catch (error: any) {
+									res.writeHead(400);
+									res.end(error.toString());
+								}
 								break;
 
 							default:
@@ -313,71 +431,101 @@ export const startServer = async (
 						}
 					} else if (req.url.startsWith(INSTALL_PATH)) {
 						switch (req.method) {
-							case "PUT":
-								getBody(req, async (body) => {
-									const name = body;
+							case "PUT": {
+								const installArgs: InstallDependency = JSON.parse(body);
 
-									let packageName = name;
-									if (name.endsWith(".tgz")) {
-										packageName = await getPackageName(name);
-									}
+								const packageName = installArgs.name; // @abc/123
+								let installName = installArgs.name; // abc123.tgz or @abc/123 (npm package name)
+								let clear: (() => void) | undefined;
+								if (installArgs.type === "tgz") {
+									const binary = fromBase64(installArgs.base64);
+									const tempFile = tmp.fileSync({
+										name:
+											base58btc.encode(Buffer.from(installName)) +
+											uuid() +
+											".tgz",
+									});
+									fs.writeFileSync(tempFile.fd, binary);
+									clear = () => tempFile.removeCallback();
+									installName = tempFile.name;
+								} else {
+									clear = undefined;
+								}
 
-									if (!name || name.length === 0) {
-										res.writeHead(400);
-										res.end("Invalid package: " + name);
-									} else {
-										const child_process = await import("child_process");
-										try {
-											// TODO do this without sudo. i.e. for servers provide arguments so that this app folder is writeable by default by the user
-											child_process.execSync(
-												`sudo npm install ${name} --prefix ${__dirname} --no-save --no-package-lock`
-											); // TODO omit=dev ? but this makes breaks the tests after running once?
-										} catch (error: any) {
+								if (!installName || installName.length === 0) {
+									res.writeHead(400);
+									res.end("Invalid package: " + packageName);
+								} else {
+									try {
+										// TODO do this without sudo. i.e. for servers provide arguments so that this app folder is writeable by default by the user
+										const installDir =
+											process.env.PEERBIT_MODULES_PATH ||
+											findPeerbitProgramFolder(__dirname);
+										let permission = "";
+										if (!installDir) {
 											res.writeHead(400);
-											res.end(
-												"Failed ot install library: " +
-													name +
-													". " +
-													error.toString()
-											);
+											res.end("Missing installation directory");
 											return;
 										}
-
 										try {
-											const programsPre = new Set(
-												getProgramFromVariants().map(
-													(x) => getSchema(x).variant
-												)
-											);
+											fs.accessSync(installDir, fs.constants.W_OK);
+										} catch (error) {
+											permission = "sudo";
+										}
 
-											await import(
-												/* webpackIgnore: true */ /* @vite-ignore */ packageName
-											);
-											const programsPost = getProgramFromVariants()?.map((x) =>
-												getSchema(x)
-											);
-											const newPrograms: { variant: string }[] = [];
-											for (const p of programsPost) {
-												if (!programsPre.has(p.variant)) {
-													newPrograms.push(p as { variant: string });
-												}
-											}
+										console.log("Installing package: " + installName);
+										execSync(
+											`${permission} npm install ${installName} --prefix ${installDir} --no-save --no-package-lock`
+										); // TODO omit=dev ? but this makes breaks the tests after running once?
+									} catch (error: any) {
+										res.writeHead(400);
+										res.end(
+											"Failed to install library: " +
+												packageName +
+												". " +
+												error.toString()
+										);
+										clear?.();
+										return;
+									}
 
-											res.writeHead(200);
-											res.end(
-												JSON.stringify(newPrograms.map((x) => x.variant))
-											);
-										} catch (e: any) {
-											res.writeHead(400);
-											res.end(e.message.toString?.());
+									try {
+										const programsPre = new Set(
+											getProgramFromVariants().map((x) => getSchema(x).variant)
+										);
+
+										await import(
+											/* webpackIgnore: true */ /* @vite-ignore */ packageName
+										);
+										await properties?.session?.imports.add(
+											packageName,
+											new Uint8Array()
+										);
+										const programsPost = getProgramFromVariants()?.map((x) =>
+											getSchema(x)
+										);
+										const newPrograms: { variant: string }[] = [];
+										for (const p of programsPost) {
+											if (!programsPre.has(p.variant)) {
+												newPrograms.push(p as { variant: string });
+											}
 										}
+
+										res.writeHead(200);
+										res.end(JSON.stringify(newPrograms.map((x) => x.variant)));
+									} catch (e: any) {
+										res.writeHead(400);
+										res.end(e.message.toString?.());
+										clear?.();
 									}
-								});
+								}
 								break;
+							}
 
-							default:
+							default: {
 								r404();
 								break;
+							}
 						}
 					} else if (req.url.startsWith(BOOTSTRAP_PATH)) {
 						switch (req.method) {
@@ -392,6 +540,48 @@ export const startServer = async (
 								res.end();
 								break;
 
+							default:
+								r404();
+								break;
+						}
+					} else if (req.url.startsWith(TRUST_PATH)) {
+						switch (req.method) {
+							case "PUT": {
+								trust.add(getPathValue(req, 1));
+								res.writeHead(200);
+								res.end();
+								break;
+							}
+							case "DELETE": {
+								const removed = trust.remove(getPathValue(req, 1));
+								res.writeHead(200);
+								res.end(removed);
+								break;
+							}
+							default:
+								r404();
+								break;
+						}
+					} else if (req.url.startsWith(RESTART_PATH)) {
+						switch (req.method) {
+							case "POST":
+								res.writeHead(200);
+								res.end();
+								restart();
+								break;
+
+							default:
+								r404();
+								break;
+						}
+					} else if (req.url.startsWith(TERMINATE_PATH)) {
+						switch (req.method) {
+							case "POST":
+								res.writeHead(200);
+								res.end();
+								process.exit(0);
+								break;
+
 							default:
 								r404();
 								break;
@@ -431,6 +621,7 @@ export const startServer = async (
 			});
 		});
 	});
+	await waitFor(() => server.listening);
 	console.log("API available at port", port);
 	return server;
 };