@mevdragon/vidfarm-devcli 0.5.3 → 0.7.0

package/dist/src/services/template-runtime-bundles.js

@@ -0,0 +1,217 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { gunzipSync } from "node:zlib";
+import { devLog } from "../lib/dev-log.js";
+import { loadTemplateFromModule } from "./template-loader.js";
+import { StorageService } from "./storage.js";
+const storage = new StorageService();
+const releaseCache = new Map();
+const MAX_EXTRACTED_RUNTIME_RELEASES = 6;
+export async function loadTemplateFromRuntimeRelease(release) {
+    const cacheKey = `${release.id}:${release.commitSha}:${release.runtimeArtifactSha256 ?? "local"}`;
+    const existing = releaseCache.get(cacheKey);
+    if (existing) {
+        return existing;
+    }
+    const loadPromise = loadRuntimeReleaseUncached(release);
+    releaseCache.set(cacheKey, loadPromise);
+    try {
+        return await loadPromise;
+    }
+    catch (error) {
+        releaseCache.delete(cacheKey);
+        throw error;
+    }
+}
+async function loadRuntimeReleaseUncached(release) {
+    if (release.runtimeArtifactKey) {
+        const checkoutPath = await extractRuntimeArtifact(release);
+        const modulePath = path.join(checkoutPath, release.runtimeModulePath ?? relativeModulePath(release));
+        devLog("template_runtime.release.loading", {
+            release_id: release.id,
+            template_id: release.templateId,
+            runtime_artifact_key: release.runtimeArtifactKey,
+            module_path: modulePath
+        });
+        const template = await loadTemplateFromModule(modulePath);
+        return {
+            ...template,
+            skillPath: path.join(checkoutPath, relativeSkillPath(release))
+        };
+    }
+    if (release.modulePath && existsSync(release.modulePath)) {
+        devLog("template_runtime.release.local_loading", {
+            release_id: release.id,
+            template_id: release.templateId,
+            module_path: release.modulePath
+        });
+        const template = await loadTemplateFromModule(release.modulePath);
+        return {
+            ...template,
+            skillPath: release.skillPath
+        };
+    }
+    throw new Error(`Active template release ${release.id} has no runtime artifact and no local module path.`);
+}
+async function extractRuntimeArtifact(release) {
+    const root = path.join(os.tmpdir(), "vidfarm-template-releases");
+    const checkoutPath = path.join(root, `${safePathSegment(release.templateId)}-${safePathSegment(release.commitSha)}`);
+    const markerPath = path.join(checkoutPath, ".vidfarm-runtime-ready");
+    if (existsSync(markerPath)) {
+        return checkoutPath;
+    }
+    const archive = await storage.readBuffer(String(release.runtimeArtifactKey));
+    if (!archive) {
+        throw new Error(`Runtime template artifact not found: ${release.runtimeArtifactKey}`);
+    }
+    const sha256 = createHash("sha256").update(archive).digest("hex");
+    if (release.runtimeArtifactSha256 && sha256 !== release.runtimeArtifactSha256) {
+        throw new Error(`Runtime template artifact hash mismatch for ${release.runtimeArtifactKey}.`);
+    }
+    try {
+        extractRuntimeArtifactToPath({ archive, root, checkoutPath, markerPath, release, sha256 });
+    }
+    catch (error) {
+        if (!isNoSpaceError(error)) {
+            throw error;
+        }
+        devLog("template_runtime.release.extract_enospc_retry", {
+            release_id: release.id,
+            template_id: release.templateId,
+            checkout_path: checkoutPath
+        });
+        pruneExtractedRuntimeArtifacts(root, checkoutPath, 0);
+        extractRuntimeArtifactToPath({ archive, root, checkoutPath, markerPath, release, sha256 });
+    }
+    return checkoutPath;
+}
+function extractRuntimeArtifactToPath(input) {
+    mkdirSync(input.root, { recursive: true });
+    pruneExtractedRuntimeArtifacts(input.root, input.checkoutPath, MAX_EXTRACTED_RUNTIME_RELEASES);
+    rmSync(input.checkoutPath, { recursive: true, force: true });
+    mkdirSync(input.checkoutPath, { recursive: true });
+    extractTarGzBuffer(input.archive, input.checkoutPath);
+    linkSharedNodeModules(input.checkoutPath);
+    writeFileSync(input.markerPath, JSON.stringify({
+        release_id: input.release.id,
+        template_id: input.release.templateId,
+        commit_sha: input.release.commitSha,
+        runtime_artifact_key: input.release.runtimeArtifactKey,
+        runtime_artifact_sha256: input.sha256,
+        extracted_at: new Date().toISOString()
+    }, null, 2));
+    devLog("template_runtime.release.extracted", {
+        release_id: input.release.id,
+        template_id: input.release.templateId,
+        checkout_path: input.checkoutPath,
+        runtime_artifact_key: input.release.runtimeArtifactKey
+    });
+}
+function pruneExtractedRuntimeArtifacts(root, keepPath, maxEntries) {
+    if (!existsSync(root)) {
+        return;
+    }
+    const keep = path.resolve(keepPath);
+    const entries = readdirSync(root, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => {
+        const fullPath = path.join(root, entry.name);
+        let mtimeMs = 0;
+        try {
+            mtimeMs = statSync(fullPath).mtimeMs;
+        }
+        catch {
+            return null;
+        }
+        return { fullPath, mtimeMs };
+    })
+        .filter((entry) => Boolean(entry))
+        .filter((entry) => path.resolve(entry.fullPath) !== keep)
+        .sort((a, b) => b.mtimeMs - a.mtimeMs);
+    const stale = maxEntries <= 0 ? entries : entries.slice(Math.max(0, maxEntries - 1));
+    for (const entry of stale) {
+        rmSync(entry.fullPath, { recursive: true, force: true });
+    }
+}
+function isNoSpaceError(error) {
+    return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOSPC");
+}
+function linkSharedNodeModules(checkoutPath) {
+    const target = path.resolve(process.env.LAMBDA_TASK_ROOT ?? process.cwd(), "node_modules");
+    const linkPath = path.join(checkoutPath, "node_modules");
+    if (!existsSync(target) || existsSync(linkPath)) {
+        return;
+    }
+    symlinkSync(target, linkPath, "dir");
+}
+function relativeModulePath(release) {
+    if (release.runtimeModulePath) {
+        return release.runtimeModulePath;
+    }
+    return path.relative(release.checkoutPath, release.modulePath).split(path.sep).join("/");
+}
+function relativeSkillPath(release) {
+    return path.relative(release.checkoutPath, release.skillPath).split(path.sep).join("/");
+}
+function safePathSegment(value) {
+    return value.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 180);
+}
+function extractTarGzBuffer(archive, outputDir) {
+    const tar = gunzipSync(archive);
+    let offset = 0;
+    while (offset + 512 <= tar.length) {
+        const header = tar.subarray(offset, offset + 512);
+        offset += 512;
+        if (header.every((byte) => byte === 0)) {
+            break;
+        }
+        const name = readTarString(header, 0, 100);
+        const prefix = readTarString(header, 345, 155);
+        const fullName = [prefix, name].filter(Boolean).join("/");
+        const size = Number.parseInt(readTarString(header, 124, 12).trim() || "0", 8) || 0;
+        const typeFlag = readTarString(header, 156, 1) || "0";
+        const body = tar.subarray(offset, offset + size);
+        offset += Math.ceil(size / 512) * 512;
+        if (!fullName || fullName === "." || typeFlag === "x" || typeFlag === "g") {
+            continue;
+        }
+        const outputPath = resolveTarOutputPath(outputDir, fullName);
+        if (!outputPath) {
+            continue;
+        }
+        if (typeFlag === "5") {
+            mkdirSync(outputPath, { recursive: true });
+            continue;
+        }
+        if (typeFlag === "2") {
+            continue;
+        }
+        if (typeFlag !== "0" && typeFlag !== "") {
+            continue;
+        }
+        mkdirSync(path.dirname(outputPath), { recursive: true });
+        writeFileSync(outputPath, body);
+    }
+}
+function readTarString(buffer, offset, length) {
+    return buffer
+        .subarray(offset, offset + length)
+        .toString("utf8")
+        .replace(/\0.*$/s, "")
+        .trim();
+}
+function resolveTarOutputPath(outputDir, entryName) {
+    const normalized = entryName.replace(/^\.\/+/, "");
+    if (!normalized || normalized.startsWith("/") || normalized.includes("..")) {
+        return null;
+    }
+    const root = path.resolve(outputDir);
+    const outputPath = path.resolve(root, normalized);
+    if (outputPath !== root && !outputPath.startsWith(`${root}${path.sep}`)) {
+        return null;
+    }
+    return outputPath;
+}
+//# sourceMappingURL=template-runtime-bundles.js.map