@hops-ops/distributed 4.8.0 → 4.9.0

package/dist/sveltekit/vite.js

@@ -1,12 +1,17 @@
 import { spawn } from 'node:child_process';
-import { existsSync, lstatSync, realpathSync } from 'node:fs';
-import { cp, lstat, mkdir, mkdtemp, open, readFile, realpath, rename, rm, unlink, writeFile } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { existsSync, lstatSync, readFileSync, realpathSync } from 'node:fs';
+import { cp, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, unlink, writeFile } from 'node:fs/promises';
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { isMainThread } from 'node:worker_threads';
 const GENERATED_SVELTEKIT_MODULE = 'sveltekit.ts';
 const MAX_COMMAND_OUTPUT_BYTES = 16 * 1024 * 1024;
+const MAX_GENERATED_COMPARE_FILES = 20_000;
+const MAX_GENERATED_COMPARE_BYTES = 128 * 1024 * 1024;
 const MODULE_NAME = /^\$distributed(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)*$/;
 const COMPILER_LOCK = join('.svelte-kit', 'distributed', 'compiler.lock');
+const GENERATION_META = 'distributed-generation';
+const MAX_LIFECYCLE_STATE_BYTES = 1024 * 1024;
 const COMPILER_COORDINATORS = Symbol.for('@hops-ops/distributed/sveltekit/compiler-coordinators/v1');
 /** Vite proxy config for GraphQL HTTP and WebSocket traffic. */
 export function distributedGraphqlProxy(options) {
@@ -22,6 +27,22 @@ export function distributedGraphqlProxy(options) {
     }
     return { [path]: { target, changeOrigin: true, ws: true } };
 }
+/** Lifecycle-only side channel for projects using committed generated clients. */
+export function distributedLifecycle() {
+    let frameworkDist;
+    return {
+        name: '@hops-ops/distributed:lifecycle',
+        enforce: 'pre',
+        configResolved(config) {
+            frameworkDist = localFrameworkDist(config.root);
+        },
+        configureServer: configureLifecycleServer,
+        transformIndexHtml: lifecycleGenerationMeta,
+        handleHotUpdate(context) {
+            return suppressFrameworkHotUpdate(context, frameworkDist);
+        }
+    };
+}
 /** Generate every configured surface through the same transaction used by Vite. */
 export async function generateDistributedSvelteKit(options) {
     await runCompilerOnce(options, 'generate');
@@ -38,12 +59,14 @@ export async function checkDistributedSvelteKit(options) {
  * coalesced, abortable, and always invoked without a shell.
  */
 export function distributedSvelteKit(options) {
+    const lifecycleOwnsCompile = process.env.DISTRIBUTED_LIFECYCLE_DIR !== undefined;
     let resolved;
     let dirty = false;
     let running;
     let completedGeneration = 0;
     let reloadedGeneration = 0;
     let stopped = false;
+    let frameworkDist;
     const children = new Set();
     const cancellation = new AbortController();
     let lock;
@@ -106,7 +129,17 @@ export function distributedSvelteKit(options) {
                 throw new Error('Distributed SvelteKit Vite plugin was configured more than once');
             }
             resolved = resolveIntegration(options, options.cwd ?? config.root);
+            frameworkDist = localFrameworkDist(config.root);
             await validateResolvedPaths(resolved);
+            /*
+             * `distributed dev` has already staged this generation and owns every
+             * compiler input through its application watcher. Recompiling here would
+             * race the API process for Cargo and mutate generated output outside the
+             * lifecycle transaction. The virtual modules still resolve the staged
+             * files, while compiler inputs below are held for the supervisor reload.
+             */
+            if (lifecycleOwnsCompile)
+                return;
             /*
              * SvelteKit post-build analysis loads Vite config in an isolated
              * worker marked with SVELTEKIT_FORK. That pass reads framework
@@ -126,18 +159,26 @@ export function distributedSvelteKit(options) {
             }
         },
         configureServer(server) {
+            configureLifecycleServer(server);
             const integration = requireResolved(resolved);
-            const roots = integration.clients.flatMap((client) => client.watchRoots);
-            if (roots.length > 0)
-                server.watcher.add(roots);
+            if (!lifecycleOwnsCompile) {
+                const roots = integration.clients.flatMap((client) => [
+                    ...client.watchRoots,
+                    ...client.manifestWatchRoots
+                ]);
+                if (roots.length > 0)
+                    server.watcher.add(roots);
+            }
             server.httpServer?.once('close', () => {
                 void stop();
             });
         },
         buildStart() {
             const integration = requireResolved(resolved);
+            if (lifecycleOwnsCompile)
+                return;
             for (const client of integration.clients) {
-                for (const root of client.watchRoots)
+                for (const root of [...client.watchRoots, ...client.manifestWatchRoots])
                     this.addWatchFile(root);
             }
         },
@@ -151,10 +192,16 @@ export function distributedSvelteKit(options) {
                 return undefined;
             return `export * from ${JSON.stringify(portablePath(client.entry))};\n`;
         },
+        transformIndexHtml: lifecycleGenerationMeta,
         async handleHotUpdate(context) {
+            const suppressed = suppressFrameworkHotUpdate(context, frameworkDist);
+            if (suppressed !== undefined)
+                return suppressed;
             const integration = requireResolved(resolved);
-            if (!isGraphqlInput(context.file, integration))
+            if (!isCompilerInput(context.file, integration))
                 return undefined;
+            if (lifecycleOwnsCompile)
+                return [];
             try {
                 await compile(`GraphQL change ${context.file}`);
             }
@@ -173,19 +220,220 @@ export function distributedSvelteKit(options) {
                     context.server.moduleGraph.invalidateModule(module);
                 }
             }
-            context.server.ws.send({ type: 'full-reload', path: '*' });
+            if (process.env.DISTRIBUTED_LIFECYCLE_DIR === undefined) {
+                context.server.ws.send({ type: 'full-reload', path: '*' });
+            }
             reloadedGeneration = completedGeneration;
             return [];
         },
         async watchChange(id) {
             const integration = requireResolved(resolved);
-            if (isGraphqlInput(id, integration)) {
+            if (lifecycleOwnsCompile)
+                return;
+            if (isCompilerInput(id, integration)) {
                 await compile(`GraphQL watch change ${id}`);
             }
         },
         closeBundle: stop
     };
 }
+function localFrameworkDist(root) {
+    try {
+        const candidate = join(root, 'node_modules', '@hops-ops', 'distributed', 'dist');
+        return existsSync(candidate) ? realpathSync(candidate) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function suppressFrameworkHotUpdate(context, frameworkDist) {
+    if (process.env.DISTRIBUTED_LIFECYCLE_DIR === undefined ||
+        frameworkDist === undefined ||
+        !isWithin(frameworkDist, canonicalExistingPath(context.file)))
+        return undefined;
+    for (const module of context.modules ?? []) {
+        context.server.moduleGraph.invalidateModule(module);
+    }
+    return [];
+}
+function canonicalExistingPath(value) {
+    try {
+        return realpathSync(value);
+    }
+    catch {
+        return resolve(value);
+    }
+}
+const CONTROL_ID = /^[A-Za-z0-9_:-]{16,128}$/;
+const MAX_ACK_BYTES = 4096;
+function lifecycleGenerationMeta() {
+    const configured = process.env.DISTRIBUTED_LIFECYCLE_DIR;
+    if (configured === undefined || !isAbsolute(configured))
+        return [];
+    try {
+        const encoded = readFileSync(join(resolve(configured), 'dev.json'));
+        if (encoded.byteLength > MAX_LIFECYCLE_STATE_BYTES)
+            return [];
+        const state = JSON.parse(encoded.toString('utf8'));
+        const generationId = state.active?.generationId;
+        if (typeof generationId !== 'string' ||
+            generationId.length === 0 ||
+            generationId.length > 512 ||
+            generationId !== generationId.trim() ||
+            /[\u0000-\u001f\u007f]/.test(generationId))
+            return [];
+        return [Object.freeze({
+                tag: 'meta',
+                attrs: Object.freeze({ name: GENERATION_META, content: generationId }),
+                injectTo: 'head-prepend'
+            })];
+    }
+    catch {
+        // A lifecycle file is atomically replaced. Missing or malformed state
+        // simply omits the hint; the browser falls back to its first poll.
+        return [];
+    }
+}
+function configureLifecycleServer(server) {
+    const configured = process.env.DISTRIBUTED_LIFECYCLE_DIR;
+    if (configured === undefined || !isAbsolute(configured))
+        return;
+    const lifecycleRoot = resolve(configured);
+    server.middlewares?.use('/__distributed/lifecycle', (request, response) => {
+        void handleLifecycleRequest(lifecycleRoot, request, response).catch(() => {
+            if (response.statusCode < 400)
+                response.statusCode = 500;
+            response.end();
+        });
+    });
+}
+async function handleLifecycleRequest(lifecycleRoot, request, response) {
+    response.setHeader('cache-control', 'no-store');
+    if (request.method === 'GET') {
+        const participant = singleHeader(request.headers['x-distributed-participant']);
+        if (participant !== undefined && CONTROL_ID.test(participant)) {
+            await writeParticipantHeartbeat(lifecycleRoot, participant).catch(() => undefined);
+        }
+        const state = await readLifecycleState(lifecycleRoot);
+        if (state === undefined) {
+            response.statusCode = 404;
+            response.end();
+            return;
+        }
+        response.statusCode = 200;
+        response.setHeader('content-type', 'application/json');
+        response.end(state);
+        return;
+    }
+    if (request.method === 'POST') {
+        const contentType = singleHeader(request.headers['content-type']);
+        if (contentType?.split(';', 1)[0]?.trim().toLowerCase() !== 'application/json') {
+            response.statusCode = 415;
+            response.end();
+            return;
+        }
+        if (!sameOriginLifecycleRequest(request)) {
+            response.statusCode = 403;
+            response.end();
+            return;
+        }
+        const body = JSON.parse(await readRequestBody(request, MAX_ACK_BYTES));
+        if (!CONTROL_ID.test(String(body.transitionId ?? '')) ||
+            !CONTROL_ID.test(String(body.participantId ?? '')) ||
+            typeof body.ok !== 'boolean') {
+            response.statusCode = 400;
+            response.end();
+            return;
+        }
+        const stateSource = await readLifecycleState(lifecycleRoot);
+        const state = stateSource === undefined ? undefined : JSON.parse(stateSource);
+        if (state?.phase !== 'preparing' || state.transitionId !== body.transitionId) {
+            response.statusCode = 409;
+            response.end();
+            return;
+        }
+        await writeControlJson(join(lifecycleRoot, 'dev-control', 'acks', String(body.transitionId)), `${String(body.participantId)}.json`, { ok: body.ok });
+        response.statusCode = 204;
+        response.end();
+        return;
+    }
+    response.statusCode = 405;
+    response.end();
+}
+function writeParticipantHeartbeat(root, participant) {
+    return writeControlJson(join(root, 'dev-control', 'participants'), `${participant}.json`, { seenAtUnixMs: Date.now() });
+}
+async function readLifecycleState(root) {
+    const path = join(root, 'dev.json');
+    let metadata;
+    try {
+        metadata = await lstat(path);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return undefined;
+        throw error;
+    }
+    if (metadata.isSymbolicLink() || !metadata.isFile() || metadata.size > MAX_LIFECYCLE_STATE_BYTES) {
+        throw new Error('invalid Distributed lifecycle state file');
+    }
+    return readFile(path, 'utf8');
+}
+async function writeControlJson(directory, name, value) {
+    await mkdir(directory, { recursive: true });
+    const temporary = join(directory, `.${name}.${process.pid}.${Date.now()}.${randomUUID()}`);
+    await writeFile(temporary, `${JSON.stringify(value)}\n`, { flag: 'wx', mode: 0o600 });
+    await rename(temporary, join(directory, name));
+}
+function sameOriginLifecycleRequest(request) {
+    const origin = singleHeader(request.headers.origin);
+    const host = singleHeader(request.headers.host);
+    if (origin === undefined || host === undefined)
+        return false;
+    try {
+        const parsed = new URL(origin);
+        const forwarded = singleHeader(request.headers['x-forwarded-proto']);
+        const socket = request.socket;
+        const protocol = forwarded === 'http' || forwarded === 'https'
+            ? `${forwarded}:`
+            : socket?.encrypted === true
+                ? 'https:'
+                : 'http:';
+        return (parsed.protocol === protocol &&
+            parsed.host === host &&
+            parsed.origin === origin);
+    }
+    catch {
+        return false;
+    }
+}
+function singleHeader(value) {
+    return typeof value === 'string' ? value : value?.[0];
+}
+function readRequestBody(request, maximum) {
+    return new Promise((resolvePromise, reject) => {
+        const chunks = [];
+        let bytes = 0;
+        let exceeded = false;
+        request.on('data', (chunk) => {
+            if (exceeded)
+                return;
+            bytes += chunk.byteLength;
+            if (bytes > maximum) {
+                exceeded = true;
+                chunks.length = 0;
+                reject(new Error('lifecycle acknowledgement exceeds bound'));
+                return;
+            }
+            chunks.push(chunk);
+        });
+        request.on('error', reject);
+        request.on('end', () => {
+            if (!exceeded)
+                resolvePromise(Buffer.concat(chunks).toString('utf8'));
+        });
+    });
+}
 async function runCompilerOnce(options, mode) {
     const integration = resolveIntegration(options, options.cwd ?? process.cwd());
     await validateResolvedPaths(integration);
@@ -287,7 +535,8 @@ function resolveIntegration(options, fallbackCwd) {
             routes: Object.freeze(routes),
             out,
             entry: join(out, GENERATED_SVELTEKIT_MODULE),
-            watchRoots: Object.freeze(documentWatchRoots(cwd, documents, out))
+            watchRoots: Object.freeze(documentWatchRoots(cwd, documents, out)),
+            manifestWatchRoots: Object.freeze(manifestWatchRoots(cwd, client.manifest))
         });
     });
     return Object.freeze({
@@ -352,6 +601,32 @@ function documentWatchRoots(cwd, documents, out) {
     }
     return [...roots].sort();
 }
+function manifestWatchRoots(cwd, manifest) {
+    const manifestPath = typeof manifest === 'string'
+        ? resolve(cwd, manifest)
+        : (() => {
+            const index = manifest.args.indexOf('--manifest-path');
+            return index === -1 || manifest.args[index + 1] === undefined
+                ? undefined
+                : resolve(cwd, manifest.args[index + 1]);
+        })();
+    if (manifestPath === undefined)
+        return [];
+    const root = dirname(manifestPath);
+    return [manifestPath, join(root, 'src'), join(root, 'crates')]
+        .filter((candidate) => existsSync(candidate))
+        .sort();
+}
+function isCompilerInput(file, integration) {
+    if (isGraphqlInput(file, integration))
+        return true;
+    const absolute = resolve(integration.cwd, file);
+    const basenameValue = basename(absolute);
+    if (!absolute.endsWith('.rs') && basenameValue !== 'Cargo.toml' && basenameValue !== 'Cargo.lock') {
+        return false;
+    }
+    return integration.clients.some((client) => client.manifestWatchRoots.some((root) => isWithin(root, absolute)));
+}
 function isGraphqlInput(file, integration) {
     const absolute = resolve(integration.cwd, file);
     if ((!absolute.endsWith('.graphql') && !absolute.endsWith('.gql')) ||
@@ -366,7 +641,8 @@ function isGraphqlInput(file, integration) {
 async function compileTransaction(integration, children, signal) {
     throwIfAborted(signal);
     await validateResolvedPaths(integration);
-    const transaction = await mkdtemp(join(integration.cwd, '.distributed-sveltekit-'));
+    const transactionRoot = await compilerTransactionRoot(integration);
+    const transaction = await mkdtemp(join(transactionRoot, '.distributed-sveltekit-'));
     const staged = [];
     try {
         for (const [index, client] of integration.clients.entries()) {
@@ -406,7 +682,7 @@ async function compileTransaction(integration, children, signal) {
                 output
             ];
             await runCommand(integration, args, children, signal);
-            await validateGeneratedEntrypoint(integration.cwd, output, client.module);
+            await validateGeneratedEntrypoint(transaction, output, client.module);
             staged.push({
                 client,
                 output,
@@ -424,7 +700,8 @@ async function compileTransaction(integration, children, signal) {
 async function checkTransaction(integration, children, signal) {
     throwIfAborted(signal);
     await validateResolvedPaths(integration);
-    const transaction = await mkdtemp(join(integration.cwd, '.distributed-sveltekit-check-'));
+    const transactionRoot = await compilerTransactionRoot(integration);
+    const transaction = await mkdtemp(join(transactionRoot, '.distributed-sveltekit-check-'));
     try {
         for (const [index, client] of integration.clients.entries()) {
             throwIfAborted(signal);
@@ -450,6 +727,14 @@ async function checkTransaction(integration, children, signal) {
         await rm(transaction, { recursive: true, force: true });
     }
 }
+async function compilerTransactionRoot(integration) {
+    const lifecycle = process.env.DISTRIBUTED_LIFECYCLE_DIR;
+    const root = lifecycle !== undefined && isAbsolute(lifecycle)
+        ? resolve(lifecycle)
+        : integration.cwd;
+    await mkdir(root, { recursive: true });
+    return root;
+}
 async function materializeManifest(integration, client, transaction, index, children, signal) {
     throwIfAborted(signal);
     if (typeof client.manifest === 'string') {
@@ -499,6 +784,9 @@ async function commitOutputs(integration, staged, signal) {
             throwIfAborted(signal);
             await mkdir(dirname(item.client.out), { recursive: true });
             await validateNearestExistingParent(integration.cwd, item.client.out);
+            if (item.hadOutput && await generatedTreesEqual(item.client.out, item.output)) {
+                continue;
+            }
             if (item.hadOutput)
                 await rename(item.client.out, item.backup);
             try {
@@ -522,6 +810,56 @@ async function commitOutputs(integration, staged, signal) {
         throw error;
     }
 }
+async function generatedTreesEqual(left, right) {
+    const budget = { files: 0, bytes: 0 };
+    const compare = async (leftDirectory, rightDirectory) => {
+        const [leftEntries, rightEntries] = await Promise.all([
+            readdir(leftDirectory, { withFileTypes: true }),
+            readdir(rightDirectory, { withFileTypes: true })
+        ]);
+        leftEntries.sort((a, b) => a.name.localeCompare(b.name));
+        rightEntries.sort((a, b) => a.name.localeCompare(b.name));
+        if (leftEntries.length !== rightEntries.length)
+            return false;
+        for (let index = 0; index < leftEntries.length; index += 1) {
+            const leftEntry = leftEntries[index];
+            const rightEntry = rightEntries[index];
+            if (leftEntry.name !== rightEntry.name ||
+                leftEntry.isDirectory() !== rightEntry.isDirectory() ||
+                leftEntry.isFile() !== rightEntry.isFile())
+                return false;
+            if (!leftEntry.isDirectory() && !leftEntry.isFile())
+                return false;
+            const leftPath = join(leftDirectory, leftEntry.name);
+            const rightPath = join(rightDirectory, rightEntry.name);
+            if (leftEntry.isDirectory()) {
+                if (!await compare(leftPath, rightPath))
+                    return false;
+                continue;
+            }
+            budget.files += 1;
+            if (budget.files > MAX_GENERATED_COMPARE_FILES)
+                return false;
+            const [leftMetadata, rightMetadata] = await Promise.all([
+                lstat(leftPath),
+                lstat(rightPath)
+            ]);
+            if (leftMetadata.size !== rightMetadata.size)
+                return false;
+            budget.bytes += leftMetadata.size;
+            if (budget.bytes > MAX_GENERATED_COMPARE_BYTES)
+                return false;
+            const [leftBytes, rightBytes] = await Promise.all([
+                readFile(leftPath),
+                readFile(rightPath)
+            ]);
+            if (!leftBytes.equals(rightBytes))
+                return false;
+        }
+        return true;
+    };
+    return compare(left, right);
+}
 async function runCommand(integration, args, children, signal) {
     throwIfAborted(signal);
     const argv = [...integration.commandArgs, ...args];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hops-ops/distributed",
-  "version": "4.8.0",
+  "version": "4.9.0",
   "description": "Typed GraphQL client, causal replica, command runtime, and framework adapters for Distributed services",
   "type": "module",
   "license": "UNLICENSED",
@@ -51,7 +51,8 @@
   },
   "dependencies": {
     "@graphql-typed-document-node/core": "^3.2.0",
-    "graphql": "^16.14.0"
+    "graphql": "^16.14.0",
+    "wasm-pack": "0.15.0"
   },
   "peerDependencies": {
     "react": "^18.2.0 || ^19.0.0",