@parity/product-lab 0.0.1-placeholder → 0.1.0

package/adapters/environment/engine/src/engine/owned-environment.js

@@ -0,0 +1,84 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { readFileSync, mkdirSync, writeFileSync, renameSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { OwnedEngineDependency, engineConfiguration } from "./owned-engine.js";
+export async function engineSessionEnvironment(steps) {
+    let current = await steps.start();
+    let closed = false;
+    return {
+        get name() { return current.name; },
+        get provider() { return current.provider; },
+        get provenance() { return current.provenance; },
+        get notices() { return current.notices; },
+        get startup() { return current.startup; },
+        get scenarios() { return current.scenarios; },
+        get sentences() { return current.sentences; },
+        get registry() { return current.registry; },
+        get navigation() { return current.navigation; },
+        get resources() { return current.resources; },
+        actors: () => current.actors(),
+        observe: (actor) => current.observe(actor),
+        subscribe: (sink) => current.subscribe?.(sink) ?? (() => { }),
+        get openApp() { return current.openApp?.bind(current); },
+        get chains() { return current.chains?.bind(current); },
+        get destinations() { return current.destinations?.bind(current); },
+        get createActor() { return current.createActor?.bind(current); },
+        get hosts() { return current.hosts?.bind(current); },
+        get fundsOf() { return current.fundsOf?.bind(current); },
+        get actorReadings() { return current.actorReadings?.bind(current); },
+        get startActor() { return current.startActor?.bind(current); },
+        get stopActor() { return current.stopActor?.bind(current); },
+        get deleteActor() { return current.deleteActor?.bind(current); },
+        get workspace() { return current.workspace; },
+        get startWorkspace() { return current.startWorkspace?.bind(current); },
+        get stopWorkspace() { return current.stopWorkspace?.bind(current); },
+        afterRun: () => current.afterRun?.() ?? Promise.resolve(),
+        async reset() {
+            await current.close();
+            await steps.rebuild();
+            current = await steps.start();
+        },
+        async close() {
+            if (closed)
+                return;
+            closed = true;
+            try {
+                await current.close();
+            }
+            finally {
+                await steps.stop();
+            }
+        },
+    };
+}
+export async function ownedEngineEnvironment(signal, options) {
+    const dependency = new OwnedEngineDependency(engineConfiguration.parse(JSON.parse(readFileSync(resolve(options.engineConfig), "utf8"))), resolve(options.reports ?? `.product-lab/owned-app-runs/${Date.now()}`), options.system);
+    const sessionFile = resolve(options.sessionFile);
+    try {
+        const start = async () => {
+            signal.throwIfAborted();
+            const session = await dependency.start();
+            signal.throwIfAborted();
+            mkdirSync(dirname(sessionFile), { recursive: true });
+            const temporary = `${sessionFile}.${process.pid}.tmp`;
+            writeFileSync(temporary, JSON.stringify(session, null, 2), { mode: 0o600 });
+            renameSync(temporary, sessionFile);
+            const environment = await options.start(sessionFile, signal);
+            environment.provider = options.provider ?? `${environment.provider} · owned Engine`;
+            return environment;
+        };
+        return await engineSessionEnvironment({
+            start,
+            async rebuild() {
+                signal.throwIfAborted();
+                await dependency.reset();
+            },
+            stop: () => dependency.stop(),
+        });
+    }
+    catch (error) {
+        await dependency.stop();
+        throw error;
+    }
+}

package/adapters/environment/engine/src/engine/persisted-data.d.ts

@@ -0,0 +1,67 @@
+import type { PortAllocator } from "@parity/product-lab/ports/system";
+import { type MissingPersistedData } from "../errors.js";
+import type { NetworkPlan } from "./network-plan.js";
+export declare function nonemptyFile(file: string): boolean;
+/**
+ * A Substrate node's base path holds a chain database: RocksDB's CURRENT naming
+ * a live manifest (`chains/<id>/db/full`), or ParityDB's metadata
+ * (`chains/<id>/paritydb/full`). Network keys and statement-store files can
+ * remain after the database itself is gone, so the directory alone proves
+ * nothing. (env-relayless checks its People node the same way; adapters do
+ * not import each other.)
+ */
+export declare function hasChainDatabase(basePath: string): boolean;
+/** The value after `flag` on a node's own side (before a collator's `--` relay-side arguments). */
+export declare function ownArgument(args: readonly string[], flag: string): string | undefined;
+/**
+ * What a resumed network would lose: every node process's chain database (its
+ * own `--base-path`; a collator's relay side follows the relay over RPC and
+ * keeps no database), the identity backend's Postgres cluster (`-D`), and the
+ * IPFS repository (`IPFS_PATH`). Empty when everything is present.
+ */
+export declare function missingPersistedData(plan: Pick<NetworkPlan, "processes">): string[];
+/** As `missingPersistedData`, each piece with its process and kind (IncompleteNetworkError carries these). */
+export declare function missingPersistedParts(plan: Pick<NetworkPlan, "processes">): MissingPersistedData[];
+/**
+ * Refuse to resume a network whose saved data is incomplete. A node started
+ * without its database would begin again from the fork's genesis (or resync
+ * from its peers) instead of resuming, silently replacing the chain history
+ * the workspace's identities were created on.
+ */
+export declare function requirePersistedData(directory: string, plan: Pick<NetworkPlan, "processes">): void;
+/** The data facts Engine's `ppn start` writes into `<PPN_HOME>/config/ports.local.env` for its services. */
+export declare const LOCAL_ENV_DATA_KEYS: readonly ["PPN_DATA_DIR", "DUB_POSTGRES_DATA_DIR"];
+/**
+ * Engine's services (the dashboard and the other zombienet custom processes,
+ * which zombienet hands no environment) read `<PPN_HOME>/config/ports.local.env`,
+ * written by the last `ppn start` on that Engine workspace. When another
+ * network was booted from the same Engine workspace since, it names that
+ * network's data directory: the dashboard then serves the other network's
+ * spawn stamp (the session check refuses it as a replaced Engine) and its
+ * files. Restate this network's own data directories there before resuming it.
+ * Returns the file when it changed. Call only once this network's ports are
+ * known free: a network running from the same Engine workspace still reads it.
+ */
+export declare function restateEngineLocalEnv(plan: Pick<NetworkPlan, "processes" | "directory">): string | undefined;
+/** The Engine checkout a saved plan's shell services run from (`<checkout>/scripts/dashboard.sh`). */
+export declare function engineCheckout(plan: Pick<NetworkPlan, "processes">): string | undefined;
+/**
+ * Engine's shell services (dashboard, identity backend and its workers) read
+ * their ports from `<checkout>/config/ports.env` when they start, and every
+ * network forked from one Engine checkout shares that file. Another network's
+ * first boot rewrites it with its own ports: resumed as is, this network's
+ * services bind that network's ports (its dashboard on the other
+ * DASHBOARD_PORT), and readiness waits on ports nothing serves. Restate this
+ * network's saved ports (engine-ports.json) there before resuming it. Keys the
+ * checkout no longer has are returned as `missing`, never written. Returns
+ * undefined when the file is already this network's, or the checkout has none.
+ * Call only once this network's ports are known free.
+ */
+export declare function restateEnginePorts(checkout: string | undefined, saved: Record<string, number>): {
+    file: string;
+    changed: string[];
+    missing: string[];
+} | undefined;
+/** What each port of a plan is for: its config/ports.env name, else the process that listens on it. */
+export declare function portNames(plan: Pick<NetworkPlan, "processes">, enginePorts?: Record<string, number>): Map<number, string>;
+export declare function requireFreePorts(allocator: PortAllocator, ports: number[], names?: ReadonlyMap<number, string>): Promise<void>;

package/adapters/environment/engine/src/engine/persisted-data.js

@@ -0,0 +1,147 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { readFileSync, existsSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { IncompleteNetworkError, PortOccupiedError } from "../errors.js";
+import { applyEnginePorts } from "./engine-ports.js";
+import { ETH_RPC_PROCESS, listenedPorts } from "./plan-ports.js";
+export function nonemptyFile(file) {
+    try {
+        const info = statSync(file);
+        return info.isFile() && info.size > 0;
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT" || code === "ENOTDIR")
+            return false;
+        throw error;
+    }
+}
+export function hasChainDatabase(basePath) {
+    const chains = join(basePath, "chains");
+    if (!existsSync(chains))
+        return false;
+    return readdirSync(chains, { withFileTypes: true }).some((chain) => {
+        if (!chain.isDirectory())
+            return false;
+        const root = join(chains, chain.name);
+        const current = join(root, "db", "full", "CURRENT");
+        if (nonemptyFile(current)) {
+            const manifest = readFileSync(current, "utf8").trim();
+            if (/^MANIFEST-\d+$/.test(manifest) && nonemptyFile(join(root, "db", "full", manifest)))
+                return true;
+        }
+        return nonemptyFile(join(root, "paritydb", "full", "metadata"));
+    });
+}
+export function ownArgument(args, flag) {
+    const end = args.indexOf("--");
+    const own = end === -1 ? args : args.slice(0, end);
+    const index = own.indexOf(flag);
+    return index === -1 ? undefined : own[index + 1];
+}
+export function missingPersistedData(plan) {
+    return missingPersistedParts(plan).map((item) => item.text);
+}
+export function missingPersistedParts(plan) {
+    const missing = [];
+    for (const { id, plan: spec } of plan.processes) {
+        const name = id.replace(/^network-/, "");
+        if (id === ETH_RPC_PROCESS)
+            continue;
+        const basePath = ownArgument(spec.args, "--base-path");
+        if (basePath !== undefined) {
+            if (!hasChainDatabase(basePath))
+                missing.push({ process: name, data: "chain database", path: basePath, text: `${name} has no chain database under ${basePath}` });
+            continue;
+        }
+        if (id === "network-postgres") {
+            const data = ownArgument(spec.args, "-D");
+            if (!data || !existsSync(join(data, "PG_VERSION")))
+                missing.push({ process: name, data: "identity database", path: data, text: `the identity database has no cluster at ${data ?? "(no -D)"}` });
+        }
+        const ipfs = spec.env?.IPFS_PATH;
+        if (id === "network-ipfs" && (!ipfs || !nonemptyFile(join(ipfs, "config"))))
+            missing.push({ process: name, data: "IPFS repository", path: ipfs, text: `IPFS has no repository at ${ipfs ?? "(no IPFS_PATH)"}` });
+    }
+    return missing;
+}
+export function requirePersistedData(directory, plan) {
+    const missing = missingPersistedParts(plan);
+    if (missing.length)
+        throw new IncompleteNetworkError(directory, missing);
+}
+export const LOCAL_ENV_DATA_KEYS = ["PPN_DATA_DIR", "DUB_POSTGRES_DATA_DIR"];
+export function restateEngineLocalEnv(plan) {
+    const home = plan.processes.map((entry) => entry.plan.env?.PPN_HOME).find((value) => !!value);
+    if (!home)
+        return undefined;
+    const file = join(home, "config", "ports.local.env");
+    if (!existsSync(file))
+        return undefined;
+    const facts = {
+        PPN_DATA_DIR: plan.directory,
+        DUB_POSTGRES_DATA_DIR: join(plan.directory, "identity-pgdata"),
+    };
+    const before = readFileSync(file, "utf8");
+    const seen = new Set();
+    const lines = before.split("\n").map((line) => {
+        const key = /^([A-Z_][A-Z0-9_]*)=/.exec(line)?.[1];
+        if (!key || !(key in facts))
+            return line;
+        seen.add(key);
+        return `${key}=${facts[key]}`;
+    });
+    const missing = LOCAL_ENV_DATA_KEYS.filter((key) => !seen.has(key)).map((key) => `${key}=${facts[key]}`);
+    if (missing.length) {
+        if (lines.at(-1) === "")
+            lines.pop();
+        lines.push(...missing, "");
+    }
+    const after = lines.join("\n");
+    if (after === before)
+        return undefined;
+    writeFileSync(file, after);
+    return file;
+}
+export function engineCheckout(plan) {
+    const scripts = plan.processes
+        .filter((entry) => entry.plan.env?.PPN_HOME)
+        .sort((a, b) => Number(b.id === "network-dashboard") - Number(a.id === "network-dashboard"))
+        .map((entry) => entry.plan.command)
+        .find((command) => command.includes("/scripts/"));
+    return scripts?.slice(0, scripts.indexOf("/scripts/"));
+}
+export function restateEnginePorts(checkout, saved) {
+    if (!checkout || !existsSync(join(checkout, "config", "ports.env")))
+        return undefined;
+    const current = applyEnginePorts(checkout, {});
+    const changed = Object.entries(saved).filter(([key, port]) => key in current && current[key] !== port);
+    const missing = Object.keys(saved).filter((key) => !(key in current));
+    if (!changed.length && !missing.length)
+        return undefined;
+    if (changed.length)
+        applyEnginePorts(checkout, Object.fromEntries(changed));
+    return {
+        file: join(checkout, "config", "ports.env"),
+        changed: changed.map(([key, port]) => `${key} ${current[key]}→${port}`),
+        missing,
+    };
+}
+export function portNames(plan, enginePorts = {}) {
+    const names = new Map();
+    for (const [key, port] of Object.entries(enginePorts))
+        names.set(port, `Engine ${key}`);
+    for (const entry of plan.processes)
+        for (const { port } of listenedPorts(entry.plan.args))
+            if (!names.has(port))
+                names.set(port, entry.id);
+    return names;
+}
+export async function requireFreePorts(allocator, ports, names = new Map()) {
+    for (const port of ports)
+        if (!(await allocator.isFree(port))) {
+            const name = names.get(port);
+            throw new PortOccupiedError(port, name);
+        }
+}

package/adapters/environment/engine/src/engine/persisted-isolation.d.ts

@@ -0,0 +1,33 @@
+import type { SyncCommandRunner } from "@parity/product-lab/ports/system";
+import { type ForkIsolation } from "./node-isolation.js";
+/** Generates and reads libp2p node keys; the secret stays in its file. */
+export interface NodeKeys {
+    /** Write a new key into `file` (mode 0600). */
+    generate(file: string): void;
+    /** The peer id of the key in `file`. */
+    peerId(file: string): string;
+}
+/** Node keys through a node binary's `key` subcommands (`polkadot`, `polkadot-omni-node`). */
+export declare function nodeKeyTool(binary: string, commands: SyncCommandRunner): NodeKeys;
+export declare const AUTHORITY_DISCOVERY_CACHE = "authority_discovery_addr_cache.json";
+export interface PersistedIsolation {
+    isolation: ForkIsolation;
+    /** Node name to its isolated arguments. */
+    args: Record<string, string[]>;
+    /** Chain spec copies whose bootNodes were rewritten. */
+    specs: string[];
+    /** Nodes that received a new key in this call. */
+    rekeyed: string[];
+    /** Authority-discovery caches removed because their node's peers changed. */
+    cachesCleared: string[];
+}
+/**
+ * Both layers on a persisted network directory (the one holding zombie.json).
+ * Keys already generated are reused, so applying it again only re-derives
+ * arguments (for example to change `expose`).
+ */
+export declare function isolatePersistedNetwork(directory: string, options: {
+    expose: boolean;
+    keys?: NodeKeys;
+    commands?: SyncCommandRunner;
+}): PersistedIsolation;

package/adapters/environment/engine/src/engine/persisted-isolation.js

@@ -0,0 +1,91 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { chmodSync, existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+import { checkedSync } from "../setup/sync-command.js";
+import { isolateNodeArgs, renameBootNodes, zombieNodes, chainSpecFiles } from "./node-isolation.js";
+export function nodeKeyTool(binary, commands) {
+    return {
+        generate(file) {
+            checkedSync(commands, { command: binary, args: ["key", "generate-node-key", "--file", file], timeoutMs: 30_000 });
+            chmodSync(file, 0o600);
+        },
+        peerId(file) {
+            const peer = checkedSync(commands, { command: binary, args: ["key", "inspect-node-key", "--file", file], timeoutMs: 30_000 }).stdout.trim();
+            if (!/^12D3Koo\w+$/.test(peer))
+                throw new Error(`Unexpected peer id from ${binary} key inspect-node-key`);
+            return peer;
+        },
+    };
+}
+export const AUTHORITY_DISCOVERY_CACHE = "authority_discovery_addr_cache.json";
+function authorityDiscoveryCaches(base) {
+    const chains = join(base, "chains");
+    if (!existsSync(chains))
+        return [];
+    return readdirSync(chains)
+        .map((chain) => join(chains, chain, "network", AUTHORITY_DISCOVERY_CACHE))
+        .filter((file) => existsSync(file));
+}
+function inside(directory, file) {
+    const path = relative(directory, file);
+    return path !== "" && !path.startsWith("..") && !path.startsWith("/");
+}
+export function isolatePersistedNetwork(directory, options) {
+    const nodes = zombieNodes(directory);
+    const relay = nodes[0];
+    if (!relay)
+        throw new Error("zombie.json lists no nodes");
+    if (!options.keys && !options.commands)
+        throw new Error("Persisted fork isolation needs synchronous commands to create node keys");
+    const keys = options.keys ?? nodeKeyTool(relay.inner.program, options.commands);
+    const specFiles = [...new Set(nodes.flatMap((node) => chainSpecFiles(node.inner.args)))];
+    for (const file of specFiles)
+        if (!inside(directory, file))
+            throw new Error(`Chain spec ${file} is outside the persisted network`);
+    const rekeyed = [];
+    const peers = {};
+    const renamed = {};
+    const keyFiles = {};
+    for (const node of nodes) {
+        if (!inside(directory, node.inner.data_dir))
+            throw new Error(`Node ${node.name} keeps its data outside the persisted network`);
+        const keyFile = join(dirname(node.inner.data_dir), "node-key");
+        if (!existsSync(keyFile)) {
+            keys.generate(keyFile);
+            rekeyed.push(node.name);
+        }
+        keyFiles[node.name] = keyFile;
+        const peer = keys.peerId(keyFile);
+        peers[node.name] = peer;
+        const previous = /\/p2p\/(\w+)$/.exec(node.multiaddr ?? "")?.[1];
+        if (previous && previous !== peer)
+            renamed[previous] = peer;
+    }
+    const args = {};
+    for (const node of nodes)
+        args[node.name] = isolateNodeArgs(node.inner.args, { expose: options.expose, nodeKeyFile: keyFiles[node.name], peers: renamed });
+    for (const file of specFiles) {
+        const content = readFileSync(file, "utf8");
+        const spec = JSON.parse(content);
+        const renamedSpec = JSON.stringify(renameBootNodes(spec, renamed), null, 2);
+        if (JSON.stringify(spec, null, 2) !== renamedSpec)
+            writeFileSync(file, renamedSpec);
+    }
+    const cachesCleared = [];
+    if (rekeyed.length)
+        for (const node of nodes)
+            for (const base of [node.inner.data_dir, node.inner.relay_data_dir])
+                if (base)
+                    for (const cache of authorityDiscoveryCaches(base)) {
+                        rmSync(cache);
+                        cachesCleared.push(cache);
+                    }
+    return {
+        isolation: { version: 1, expose: options.expose, peers },
+        args,
+        specs: specFiles,
+        rekeyed,
+        cachesCleared,
+    };
+}