@deepnote/runtime-core 0.1.0

package/dist/index.cjs

@@ -0,0 +1,619 @@
+//#region rolldown:runtime
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+	if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
+		key = keys[i];
+		if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
+			get: ((k) => from[k]).bind(null, key),
+			enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+		});
+	}
+	return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
+	value: mod,
+	enumerable: true
+}) : target, mod));
+
+//#endregion
+let node_fs_promises = require("node:fs/promises");
+node_fs_promises = __toESM(node_fs_promises);
+let __deepnote_blocks = require("@deepnote/blocks");
+__deepnote_blocks = __toESM(__deepnote_blocks);
+let __jupyterlab_services = require("@jupyterlab/services");
+__jupyterlab_services = __toESM(__jupyterlab_services);
+let node_child_process = require("node:child_process");
+node_child_process = __toESM(node_child_process);
+let tcp_port_used = require("tcp-port-used");
+tcp_port_used = __toESM(tcp_port_used);
+let node_path = require("node:path");
+node_path = __toESM(node_path);
+
+//#region src/kernel-client.ts
+/**
+* Client for communicating with a Jupyter kernel via the Jupyter protocol.
+*/
+var KernelClient = class {
+	kernelManager = null;
+	sessionManager = null;
+	session = null;
+	kernel = null;
+	/**
+	* Connect to a Jupyter server and start a kernel session.
+	*/
+	async connect(serverUrl) {
+		try {
+			const url = new URL(serverUrl);
+			url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+			const wsUrl = url.toString();
+			const serverSettings = __jupyterlab_services.ServerConnection.makeSettings({
+				baseUrl: serverUrl,
+				wsUrl
+			});
+			this.kernelManager = new __jupyterlab_services.KernelManager({ serverSettings });
+			this.sessionManager = new __jupyterlab_services.SessionManager({
+				kernelManager: this.kernelManager,
+				serverSettings
+			});
+			await this.sessionManager.ready;
+			this.session = await this.sessionManager.startNew({
+				name: "deepnote-cli",
+				path: "deepnote-cli",
+				type: "notebook",
+				kernel: { name: "python3" }
+			});
+			this.kernel = this.session.kernel;
+			if (!this.kernel) throw new Error("Failed to start kernel");
+			await this.waitForKernelIdle();
+		} catch (error) {
+			await this.disconnect();
+			throw error;
+		}
+	}
+	/**
+	* Wait for the kernel to reach idle status.
+	*/
+	async waitForKernelIdle(timeoutMs = 3e4) {
+		if (!this.kernel) return;
+		const startTime = Date.now();
+		while (this.kernel.status !== "idle") {
+			if (Date.now() - startTime > timeoutMs) throw new Error(`Kernel failed to reach idle status within ${timeoutMs}ms. Current status: ${this.kernel.status}`);
+			if (this.kernel.status === "dead") throw new Error("Kernel is dead");
+			await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+	}
+	/**
+	* Execute code on the kernel and collect outputs.
+	*/
+	async execute(code, callbacks) {
+		if (!this.kernel) throw new Error("Kernel not connected. Call connect() first.");
+		return new Promise((resolve, reject) => {
+			const outputs = [];
+			let executionCount = null;
+			const future = this.kernel?.requestExecute({ code });
+			if (!future) {
+				reject(/* @__PURE__ */ new Error("Failed to execute code on kernel"));
+				return;
+			}
+			callbacks?.onStart?.();
+			future.onIOPub = (msg) => {
+				const msgType = msg.header.msg_type;
+				if (msgType === "execute_input") executionCount = msg.content.execution_count ?? null;
+				else if ([
+					"stream",
+					"execute_result",
+					"display_data",
+					"error"
+				].includes(msgType)) {
+					const output = this.messageToOutput(msg);
+					outputs.push(output);
+					callbacks?.onOutput?.(output);
+				}
+			};
+			future.done.then(() => {
+				const result = {
+					success: !outputs.some((o) => o.output_type === "error"),
+					outputs,
+					executionCount
+				};
+				callbacks?.onDone?.(result);
+				resolve(result);
+			}).catch(reject).finally(() => future?.dispose());
+		});
+	}
+	/**
+	* Disconnect from the kernel and clean up resources.
+	*/
+	async disconnect() {
+		if (this.session) {
+			try {
+				await this.session.shutdown();
+			} catch {}
+			this.session.dispose();
+			this.session = null;
+		}
+		if (this.sessionManager) {
+			this.sessionManager.dispose();
+			this.sessionManager = null;
+		}
+		if (this.kernelManager) {
+			this.kernelManager.dispose();
+			this.kernelManager = null;
+		}
+		this.kernel = null;
+	}
+	/**
+	* Convert a Jupyter message to an IOutput object.
+	*/
+	messageToOutput(msg) {
+		const msgType = msg.header.msg_type;
+		const content = msg.content;
+		switch (msgType) {
+			case "stream": return {
+				output_type: "stream",
+				name: content.name,
+				text: content.text
+			};
+			case "execute_result": return {
+				output_type: "execute_result",
+				data: content.data,
+				metadata: content.metadata ?? {},
+				execution_count: content.execution_count
+			};
+			case "display_data": return {
+				output_type: "display_data",
+				data: content.data,
+				metadata: content.metadata ?? {}
+			};
+			case "error": return {
+				output_type: "error",
+				ename: content.ename,
+				evalue: content.evalue,
+				traceback: content.traceback
+			};
+			default: return {
+				output_type: "error",
+				ename: "UnknownMsgType",
+				evalue: `Received unknown message type: ${msgType}`,
+				traceback: []
+			};
+		}
+	}
+};
+
+//#endregion
+//#region src/python-env.ts
+const PYTHON_EXECUTABLES_UNIX = ["python", "python3"];
+const PYTHON_EXECUTABLES_WIN = ["python.exe", "python3.exe"];
+/**
+* Resolves the Python executable path using smart detection.
+*
+* Accepts multiple input formats (similar to uv):
+* - 'python' or 'python3' → uses system Python
+* - '/path/to/python' (executable file) → uses it directly
+* - '/path/to/venv/bin' (directory with python) → uses python from that directory
+* - '/path/to/venv' (venv root with bin/python) → uses bin/python
+*
+* @param pythonPath - Path to Python executable, bin directory, or venv root
+* @returns The resolved path to the Python executable
+* @throws Error if the path doesn't exist or no Python executable is found
+*/
+async function resolvePythonExecutable(pythonPath) {
+	if (pythonPath === "python" || pythonPath === "python3") return pythonPath;
+	let fileStat;
+	try {
+		fileStat = await (0, node_fs_promises.stat)(pythonPath);
+	} catch (err) {
+		const error = err;
+		if (error.code === "ENOENT") throw new Error(`Python path not found: ${pythonPath}`);
+		throw new Error(`Failed to access Python path: ${pythonPath} (${error.code}: ${error.message})`);
+	}
+	if (fileStat.isFile()) {
+		if ((0, node_path.basename)(pythonPath).toLowerCase().startsWith("python")) return pythonPath;
+		throw new Error(`Path is a file but doesn't appear to be a Python executable: ${pythonPath}\nExpected a file named python, python3, python.exe, or similar.`);
+	}
+	if (!fileStat.isDirectory()) throw new Error(`Python path is neither a file nor a directory: ${pythonPath}`);
+	const candidates = process.platform === "win32" ? PYTHON_EXECUTABLES_WIN : PYTHON_EXECUTABLES_UNIX;
+	const directPython = await findPythonInDirectory(pythonPath, candidates);
+	if (directPython) return directPython;
+	const binDir = process.platform === "win32" ? (0, node_path.join)(pythonPath, "Scripts") : (0, node_path.join)(pythonPath, "bin");
+	if ((await (0, node_fs_promises.stat)(binDir).catch(() => null))?.isDirectory()) {
+		const venvPython = await findPythonInDirectory(binDir, candidates);
+		if (venvPython) return venvPython;
+	}
+	const searchedPaths = [...candidates.map((c) => (0, node_path.join)(pythonPath, c)), ...candidates.map((c) => (0, node_path.join)(binDir, c))];
+	throw new Error(`No Python executable found at: ${pythonPath}\n\nSearched for:\n${searchedPaths.map((p) => `  - ${p}`).join("\n")}\n\nYou can pass:
+  - A Python executable: --python /path/to/venv/bin/python
+  - A bin directory: --python /path/to/venv/bin
+  - A venv root: --python /path/to/venv`);
+}
+/**
+* Finds a Python executable in the given directory.
+*/
+async function findPythonInDirectory(dir, candidates) {
+	for (const candidate of candidates) {
+		const pythonPath = (0, node_path.join)(dir, candidate);
+		if ((await (0, node_fs_promises.stat)(pythonPath).catch(() => null))?.isFile()) return pythonPath;
+	}
+	return null;
+}
+/**
+* Detects the default Python command available on the system.
+* Tries 'python' first, then falls back to 'python3'.
+*
+* @returns 'python' or 'python3' depending on what's available
+* @throws Error if neither python nor python3 is found
+*/
+function detectDefaultPython() {
+	if (isPythonAvailable("python")) return "python";
+	if (isPythonAvailable("python3")) return "python3";
+	throw new Error("No Python executable found.\n\nPlease ensure Python is installed and available in your PATH,\nor specify the path explicitly with --python <path>");
+}
+/**
+* Checks if a Python command is available on the system.
+*/
+function isPythonAvailable(command) {
+	try {
+		(0, node_child_process.execSync)(`${command} --version`, { stdio: "ignore" });
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+//#endregion
+//#region src/server-starter.ts
+const DEFAULT_PORT = 8888;
+const SERVER_STARTUP_TIMEOUT_MS = 12e4;
+const HEALTH_CHECK_INTERVAL_MS = 200;
+/**
+* Start the deepnote-toolkit Jupyter server.
+* Spawns `python -m deepnote_toolkit server` and waits for it to be ready.
+*/
+async function startServer(options) {
+	const { pythonEnv, workingDirectory, port, startupTimeoutMs = SERVER_STARTUP_TIMEOUT_MS } = options;
+	const pythonPath = await resolvePythonExecutable(pythonEnv);
+	const jupyterPort = await findConsecutiveAvailablePorts(port ?? DEFAULT_PORT);
+	const lspPort = jupyterPort + 1;
+	const env = { ...process.env };
+	env.DEEPNOTE_RUNTIME__RUNNING_IN_DETACHED_MODE = "true";
+	env.DEEPNOTE_ENFORCE_PIP_CONSTRAINTS = "true";
+	const serverProcess = (0, node_child_process.spawn)(pythonPath, [
+		"-m",
+		"deepnote_toolkit",
+		"server",
+		"--jupyter-port",
+		String(jupyterPort),
+		"--ls-port",
+		String(lspPort)
+	], {
+		cwd: workingDirectory,
+		env,
+		stdio: [
+			"ignore",
+			"pipe",
+			"pipe"
+		]
+	});
+	const serverInfo = {
+		url: `http://localhost:${jupyterPort}`,
+		jupyterPort,
+		lspPort,
+		process: serverProcess
+	};
+	let stdout = "";
+	let stderr = "";
+	serverProcess.stdout?.on("data", (data) => {
+		stdout += data.toString();
+		if (stdout.length > 5e3) stdout = stdout.slice(-5e3);
+	});
+	serverProcess.stderr?.on("data", (data) => {
+		stderr += data.toString();
+		if (stderr.length > 5e3) stderr = stderr.slice(-5e3);
+	});
+	const exitPromise = new Promise((_, reject) => {
+		serverProcess.on("exit", (code, signal) => {
+			reject(/* @__PURE__ */ new Error(`Server process exited unexpectedly (code=${code}, signal=${signal}).\nstderr: ${stderr}`));
+		});
+	});
+	try {
+		await Promise.race([waitForServer(serverInfo, startupTimeoutMs), exitPromise]);
+	} catch (error) {
+		serverProcess.kill("SIGKILL");
+		throw error;
+	}
+	return serverInfo;
+}
+/**
+* Stop the deepnote-toolkit server.
+*/
+async function stopServer(info) {
+	if (info.process.exitCode !== null) return;
+	info.process.kill("SIGTERM");
+	await new Promise((resolve) => {
+		const timeout = setTimeout(() => {
+			if (info.process.exitCode === null) info.process.kill("SIGKILL");
+			resolve();
+		}, 2e3);
+		info.process.once("exit", () => {
+			clearTimeout(timeout);
+			resolve();
+		});
+	});
+}
+/**
+* Find two consecutive available ports starting from the given port.
+*/
+async function findConsecutiveAvailablePorts(startPort) {
+	const maxAttempts = 100;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		const candidatePort = startPort + attempt * 2;
+		const [portInUse, nextPortInUse] = await Promise.all([isPortInUse(candidatePort), isPortInUse(candidatePort + 1)]);
+		if (!portInUse && !nextPortInUse) return candidatePort;
+	}
+	throw new Error(`Could not find consecutive available ports after ${maxAttempts} attempts starting from ${startPort}`);
+}
+/**
+* Check if a port is in use.
+*/
+async function isPortInUse(port) {
+	try {
+		return await tcp_port_used.default.check(port, "127.0.0.1");
+	} catch {
+		return false;
+	}
+}
+/**
+* Wait for the server to respond to health checks.
+*/
+async function waitForServer(info, timeoutMs) {
+	const startTime = Date.now();
+	while (Date.now() - startTime < timeoutMs) {
+		try {
+			if ((await fetch(`${info.url}/api`)).ok) return;
+		} catch {}
+		await sleep(HEALTH_CHECK_INTERVAL_MS);
+	}
+	throw new Error(`Server failed to start within ${timeoutMs}ms at ${info.url}`);
+}
+function sleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+//#endregion
+//#region src/execution-engine.ts
+const executableBlockTypes = [
+	"code",
+	"sql",
+	"notebook-function",
+	"visualization",
+	"button",
+	"big-number",
+	"input-text",
+	"input-textarea",
+	"input-checkbox",
+	"input-select",
+	"input-slider",
+	"input-date",
+	"input-date-range",
+	"input-file"
+];
+const executableBlockTypeSet = new Set(executableBlockTypes);
+/**
+* High-level execution engine for running Deepnote projects.
+*
+* @example
+* ```typescript
+* const engine = new ExecutionEngine({
+*   pythonEnv: '/path/to/venv',  // or 'python' for system Python
+*   workingDirectory: '/path/to/project',
+* })
+*
+* await engine.start()
+* try {
+*   const summary = await engine.runFile('./project.deepnote')
+*   console.log(`Executed ${summary.executedBlocks} blocks`)
+* } finally {
+*   await engine.stop()
+* }
+* ```
+*/
+var ExecutionEngine = class {
+	server = null;
+	kernel = null;
+	constructor(config) {
+		this.config = config;
+	}
+	/**
+	* Get the Jupyter server port (available after start() is called).
+	*/
+	get serverPort() {
+		return this.server?.jupyterPort ?? null;
+	}
+	/**
+	* Start the deepnote-toolkit server and connect to the kernel.
+	*/
+	async start() {
+		this.server = await startServer({
+			pythonEnv: this.config.pythonEnv,
+			workingDirectory: this.config.workingDirectory,
+			port: this.config.serverPort
+		});
+		try {
+			this.kernel = new KernelClient();
+			await this.kernel.connect(this.server.url);
+		} catch (error) {
+			await this.stop();
+			throw error;
+		}
+	}
+	/**
+	* Stop the server and disconnect from the kernel.
+	*/
+	async stop() {
+		if (this.kernel) {
+			await this.kernel.disconnect();
+			this.kernel = null;
+		}
+		if (this.server) {
+			await stopServer(this.server);
+			this.server = null;
+		}
+	}
+	/**
+	* Run a .deepnote file.
+	*/
+	async runFile(filePath, options = {}) {
+		const file = (0, __deepnote_blocks.deserializeDeepnoteFile)((0, __deepnote_blocks.decodeUtf8NoBom)(await (0, node_fs_promises.readFile)(filePath)));
+		return this.runProject(file, options);
+	}
+	/**
+	* Run a parsed DeepnoteFile.
+	*/
+	async runProject(file, options = {}) {
+		if (!this.kernel) throw new Error("Engine not started. Call start() first.");
+		if (options.inputs && Object.keys(options.inputs).length > 0) await this.injectInputs(options.inputs);
+		const startTime = Date.now();
+		let executedBlocks = 0;
+		let failedBlocks = 0;
+		const notebooks = options.notebookName ? file.project.notebooks.filter((n) => n.name === options.notebookName) : file.project.notebooks;
+		if (options.notebookName && notebooks.length === 0) throw new Error(`Notebook "${options.notebookName}" not found in project`);
+		const blockIdFilter = options.blockIds ? new Set(options.blockIds) : options.blockId ? new Set([options.blockId]) : null;
+		const allExecutableBlocks = [];
+		for (const notebook of notebooks) {
+			const sortedBlocks = this.sortBlocks(notebook.blocks);
+			for (const block of sortedBlocks) if ((0, __deepnote_blocks.isExecutableBlock)(block)) {
+				if (blockIdFilter && !blockIdFilter.has(block.id)) continue;
+				allExecutableBlocks.push({
+					block,
+					notebookName: notebook.name
+				});
+			}
+		}
+		if (options.blockIds && allExecutableBlocks.length === 0 && options.blockIds.length > 0) for (const blockId of options.blockIds) this.assertExecutableBlockExists(blockId, notebooks);
+		const primaryBlockId = options.blockIds ? void 0 : options.blockId;
+		if (primaryBlockId && allExecutableBlocks.length === 0) this.assertExecutableBlockExists(primaryBlockId, notebooks);
+		const totalBlocks = allExecutableBlocks.length;
+		for (let i = 0; i < allExecutableBlocks.length; i++) {
+			const { block } = allExecutableBlocks[i];
+			const blockStart = Date.now();
+			await options.onBlockStart?.(block, i, totalBlocks);
+			try {
+				const code = (0, __deepnote_blocks.createPythonCode)(block);
+				const result = await this.kernel.execute(code, { onOutput: (output) => options.onOutput?.(block.id, output) });
+				const blockResult = {
+					blockId: block.id,
+					blockType: block.type,
+					success: result.success,
+					outputs: result.outputs,
+					executionCount: result.executionCount,
+					durationMs: Date.now() - blockStart
+				};
+				await options.onBlockDone?.(blockResult);
+				executedBlocks++;
+				if (!result.success) {
+					failedBlocks++;
+					break;
+				}
+			} catch (error) {
+				failedBlocks++;
+				executedBlocks++;
+				const blockResult = {
+					blockId: block.id,
+					blockType: block.type,
+					success: false,
+					outputs: [],
+					executionCount: null,
+					durationMs: Date.now() - blockStart,
+					error: error instanceof Error ? error : new Error(String(error))
+				};
+				await options.onBlockDone?.(blockResult);
+				break;
+			}
+		}
+		return {
+			totalBlocks,
+			executedBlocks,
+			failedBlocks,
+			totalDurationMs: Date.now() - startTime
+		};
+	}
+	/**
+	* Sort blocks by their sortingKey.
+	*/
+	sortBlocks(blocks) {
+		return [...blocks].sort((a, b) => a.sortingKey.localeCompare(b.sortingKey));
+	}
+	/**
+	* Ensure a requested block exists in the selected notebooks and is executable.
+	*/
+	assertExecutableBlockExists(blockId, notebooks) {
+		for (const notebook of notebooks) {
+			const block = notebook.blocks.find((b) => b.id === blockId);
+			if (!block) continue;
+			if (!(0, __deepnote_blocks.isExecutableBlock)(block)) throw new Error(`Block "${blockId}" is not executable (type: ${block.type}).`);
+			return;
+		}
+		throw new Error(`Block "${blockId}" not found in project`);
+	}
+	/**
+	* Check if a string is a valid Python identifier.
+	* Python identifiers must start with a letter or underscore,
+	* followed by letters, digits, or underscores.
+	*/
+	isValidPythonIdentifier(name) {
+		return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name);
+	}
+	/**
+	* Inject input values into the kernel before execution.
+	* Converts values to Python literals and executes assignment statements.
+	*/
+	async injectInputs(inputs) {
+		if (!this.kernel) throw new Error("Engine not started. Call start() first.");
+		const assignments = [];
+		for (const [name, value] of Object.entries(inputs)) {
+			if (!this.isValidPythonIdentifier(name)) throw new Error(`Invalid variable name: "${name}". Must be a valid Python identifier.`);
+			const pythonValue = this.toPythonLiteral(value);
+			assignments.push(`${name} = ${pythonValue}`);
+		}
+		if (assignments.length > 0) {
+			const code = assignments.join("\n");
+			const result = await this.kernel.execute(code);
+			if (!result.success) {
+				const errorOutput = result.outputs.find((o) => o.output_type === "error");
+				const errorMsg = errorOutput && "evalue" in errorOutput ? String(errorOutput.evalue) : "Failed to inject inputs";
+				throw new Error(`Failed to set input values: ${errorMsg}`);
+			}
+		}
+	}
+	/**
+	* Convert a JavaScript value to a Python literal.
+	*/
+	toPythonLiteral(value) {
+		if (value === null || value === void 0) return "None";
+		if (typeof value === "boolean") return value ? "True" : "False";
+		if (typeof value === "number") {
+			if (!Number.isFinite(value)) throw new Error(`Cannot convert non-finite number to Python: ${value}`);
+			return String(value);
+		}
+		if (typeof value === "string") return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t").replace(/\0/g, "\\x00").replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, (char) => `\\x${char.charCodeAt(0).toString(16).padStart(2, "0")}`)}'`;
+		if (Array.isArray(value)) return `[${value.map((v) => this.toPythonLiteral(v)).join(", ")}]`;
+		if (typeof value === "object") return `{${Object.entries(value).map(([k, v]) => `${this.toPythonLiteral(k)}: ${this.toPythonLiteral(v)}`).join(", ")}}`;
+		throw new Error(`Cannot convert value of type ${typeof value} to Python literal`);
+	}
+};
+
+//#endregion
+exports.ExecutionEngine = ExecutionEngine;
+exports.KernelClient = KernelClient;
+exports.detectDefaultPython = detectDefaultPython;
+exports.executableBlockTypeSet = executableBlockTypeSet;
+exports.executableBlockTypes = executableBlockTypes;
+exports.resolvePythonExecutable = resolvePythonExecutable;
+exports.startServer = startServer;
+exports.stopServer = stopServer;