ima2-gen 3.13.1 → 3.14.0

package/lib/nodeStore.js

@@ -1,15 +1,19 @@
-import { writeFile, readFile, access, mkdir } from "fs/promises";
-import { join, resolve, sep } from "path";
+import { writeFile, readFile, access, mkdir, realpath } from "fs/promises";
+import { resolve, sep } from "path";
 import { randomBytes } from "crypto";
 import { config } from "../config.js";
-import { embedImageMetadataBestEffort } from "./imageMetadataStore.js";
+import { embedImageMetadataBestEffort, isSupportedMetadataFormat } from "./imageMetadataStore.js";
 import { invalidateHistoryIndex } from "./historyIndex.js";
 export function newNodeId() {
     return "n_" + randomBytes(config.ids.nodeHexBytes).toString("hex");
 }
 export async function saveNode(rootDir, { nodeId, b64, meta, ext = "png", generatedDir = config.storage.generatedDir }) {
-    void rootDir;
+    if (typeof ext !== "string" || !isSupportedMetadataFormat(ext)) {
+        throw Object.assign(new Error("Format must be png, jpeg, jpg, or webp."), { code: "INVALID_FORMAT", status: 400 });
+    }
     const filename = `${nodeId}.${ext}`;
+    const imagePath = resolveGeneratedPath(rootDir, filename, generatedDir);
+    const metadataPath = resolveGeneratedPath(rootDir, filename + ".json", generatedDir);
     await mkdir(generatedDir, { recursive: true });
     const imageMeta = {
         ...meta,
@@ -22,8 +26,8 @@ export async function saveNode(rootDir, { nodeId, b64, meta, ext = "png", genera
     if (!embedded.embedded) {
         console.warn("[nodeStore] metadata embed skipped:", embedded.warning);
     }
-    await writeFile(join(generatedDir, filename), embedded.buffer);
-    await writeFile(join(generatedDir, filename + ".json"), JSON.stringify(meta, null, 2));
+    await writeFile(imagePath, embedded.buffer);
+    await writeFile(metadataPath, JSON.stringify(meta, null, 2));
     invalidateHistoryIndex();
     return { filename };
 }
@@ -38,13 +42,15 @@ export async function loadNodeB64(rootDir, filename, generatedDir = config.stora
         err.status = 404;
         throw err;
     }
-    const buf = await readFile(p);
+    const containedPath = await resolveCanonicalGeneratedPath(rootDir, p, generatedDir);
+    const buf = await readFile(containedPath);
     return buf.toString("base64");
 }
 export async function loadNodeMeta(rootDir, nodeId, ext = "png", generatedDir = config.storage.generatedDir) {
-    void rootDir;
     try {
-        return JSON.parse(await readFile(join(generatedDir, `${nodeId}.${ext}.json`), "utf-8"));
+        const metadataPath = resolveGeneratedPath(rootDir, `${nodeId}.${ext}.json`, generatedDir);
+        const containedPath = await resolveCanonicalGeneratedPath(rootDir, metadataPath, generatedDir);
+        return JSON.parse(await readFile(containedPath, "utf-8"));
     }
     catch {
         return null;
@@ -61,9 +67,16 @@ export async function loadAssetB64(rootDir, externalSrc, generatedDir = config.s
         err.status = 404;
         throw err;
     }
-    const buf = await readFile(p);
+    const containedPath = await resolveCanonicalGeneratedPath(rootDir, p, generatedDir);
+    const buf = await readFile(containedPath);
     return buf.toString("base64");
 }
+async function resolveCanonicalGeneratedPath(rootDir, path, generatedDir) {
+    const canonicalRoot = await realpath(generatedDir);
+    const canonicalPath = await realpath(path);
+    // Reuse the explicit-dir boundary after resolving directory and leaf symlinks.
+    return resolveGeneratedPath(rootDir, canonicalPath, canonicalRoot);
+}
 function resolveGeneratedPath(rootDir, relPath, generatedDir = config.storage.generatedDir) {
     void rootDir;
     if (typeof relPath !== "string" || relPath.length === 0) {
@@ -74,7 +87,8 @@ function resolveGeneratedPath(rootDir, relPath, generatedDir = config.storage.ge
     }
     const baseDir = resolve(generatedDir);
     const target = resolve(baseDir, relPath);
-    if (target !== baseDir && !target.startsWith(baseDir + sep)) {
+    const prefix = baseDir.endsWith(sep) ? baseDir : baseDir + sep;
+    if (target !== baseDir && !target.startsWith(prefix)) {
         const err = new Error(`Asset path escapes generated/: ${relPath}`);
         err.code = "NODE_SOURCE_INVALID";
         err.status = 400;

package/lib/nodeValidation.js

@@ -1,10 +1,14 @@
 import { validateAndNormalizeRefs } from "./refs.js";
 import { validateModeration } from "./routeHelpers.js";
 import { validateGenerationPrompt } from "./generationInputValidation.js";
-export function validateNodeInputs(ctx, prompt, references, moderation) {
+import { isSupportedMetadataFormat } from "./imageMetadataStore.js";
+export function validateNodeInputs(ctx, prompt, references, moderation, format = "png") {
     const promptError = validateGenerationPrompt(prompt);
     if (promptError)
         return promptError;
+    if (typeof format !== "string" || !isSupportedMetadataFormat(format)) {
+        return { error: { code: "INVALID_FORMAT", message: "Format must be png, jpeg, jpg, or webp." } };
+    }
     const refCheckResult = validateAndNormalizeRefs(references);
     if (refCheckResult.error) {
         return {

package/lib/pinnedHttpGet.js

@@ -0,0 +1,174 @@
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+import { resolvePublicDownloadTarget } from "./grokImageDownloadPolicy.js";
+export class PinnedGetFailure extends Error {
+}
+function pinnedLookup(target) {
+    const hostname = target.url.hostname.replace(/^\[|\]$/g, "");
+    return (host, options, callback) => {
+        const family = options.family === "IPv4" ? 4 : options.family === "IPv6" ? 6 : options.family || 0;
+        const addresses = target.addresses.filter((address) => !family || address.family === family);
+        const first = addresses[0];
+        if (host !== hostname || !first || (family !== 0 && family !== 4 && family !== 6)) {
+            callback(new PinnedGetFailure("Image download address selection failed"), []);
+            return;
+        }
+        if (options.all)
+            callback(null, addresses.map((address) => ({ ...address })));
+        else
+            callback(null, first.address, first.family);
+    };
+}
+/** Own error-event handling independently of the already-settled header promise. */
+class PinnedGetLifecycle {
+    signal;
+    requestUrl;
+    resolve;
+    reject;
+    request;
+    response;
+    stopped = false;
+    headersReceived = false;
+    onAbort;
+    constructor(signal, requestUrl, resolve, reject) {
+        this.signal = signal;
+        this.requestUrl = requestUrl;
+        this.resolve = resolve;
+        this.reject = reject;
+        this.onAbort = () => {
+            this.reject(signal.reason);
+            this.cancel(signal.reason instanceof Error ? signal.reason : undefined);
+        };
+        signal.addEventListener("abort", this.onAbort, { once: true });
+        if (signal.aborted)
+            this.onAbort();
+    }
+    cancel = (reason) => {
+        if (this.stopped)
+            return;
+        this.stopped = true;
+        this.signal.removeEventListener("abort", this.onAbort);
+        this.response?.destroy(reason);
+        this.request?.destroy(reason);
+    };
+    fail = (error) => {
+        this.reject(error); // Raw pre-header reset/EPIPE must reach the real retry classifier.
+        this.cancel(error instanceof Error ? error : undefined);
+    };
+    attach(request) {
+        this.request = request;
+        request.on("error", this.fail);
+        // Keep the response handler available for a late response even after abort.
+        request.on("response", this.accept);
+        request.once("close", () => {
+            request.removeListener("error", this.fail);
+            if (!this.headersReceived && !this.stopped) {
+                this.fail(new PinnedGetFailure("Image download closed before response"));
+            }
+        });
+        if (this.stopped)
+            request.destroy();
+    }
+    accept = (response) => {
+        const onError = (error) => this.fail(error);
+        response.on("error", onError);
+        response.once("close", () => response.removeListener("error", onError));
+        if (this.stopped || this.signal.aborted || this.headersReceived) {
+            response.destroy();
+            return;
+        }
+        this.response = response;
+        this.headersReceived = true;
+        this.resolve({
+            url: this.requestUrl,
+            status: response.statusCode ?? 0,
+            headers: { get: (name) => {
+                    const value = response.headers[name.toLowerCase()];
+                    return Array.isArray(value) ? value.join(", ") : value ?? null;
+                } },
+            body: response,
+            cancel: this.cancel,
+        });
+    };
+}
+export async function pinnedHttpGet(target, signal, options = {}) {
+    signal.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const lifecycle = new PinnedGetLifecycle(signal, target.url.href, resolve, reject);
+        try {
+            if (lifecycle.stopped)
+                return;
+            const hostname = target.url.hostname.replace(/^\[|\]$/g, "");
+            const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
+            lifecycle.attach(request(target.url, {
+                method: "GET", agent: false, signal,
+                headers: { ...options.headers, Host: target.url.host },
+                ...(options.family ? { family: options.family } : {}),
+                lookup: pinnedLookup(target),
+                ...(isIP(hostname) ? {} : { servername: hostname }),
+            }));
+            if (!lifecycle.stopped)
+                lifecycle.request?.end();
+        }
+        catch (error) {
+            lifecycle.fail(error);
+        }
+    });
+}
+const MAX_PUBLIC_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+/** Public consumers cannot opt into Grok's configured proxy exception. */
+export async function publicPinnedHttpGet(rawUrl, signal, validateUrl, headers) {
+    let url = new URL(rawUrl);
+    try {
+        for (let hop = 0;; hop++) {
+            signal.throwIfAborted();
+            validateUrl(url.href); // Includes scheme/allowhost, before DNS and every socket.
+            const target = await resolvePublicDownloadTarget(url, signal);
+            const response = await pinnedHttpGet(target, signal, headers ? { headers } : {});
+            if (!REDIRECT_STATUSES.has(response.status))
+                return response;
+            try {
+                const location = response.headers.get("location");
+                if (!location?.trim() || hop >= MAX_PUBLIC_REDIRECTS)
+                    throw new Error("PINNED_GET_REDIRECT_REFUSED");
+                url = new URL(location, url);
+            }
+            finally {
+                response.cancel();
+            }
+        }
+    }
+    catch (error) {
+        throw error;
+    }
+}
+export class PinnedBodyTooLarge extends Error {
+    constructor() { super("Remote file is too large"); }
+}
+/** Bound bytes while streaming, including absent or dishonest Content-Length. */
+export async function readPinnedBody(response, maxBytes) {
+    try {
+        if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
+            throw new PinnedBodyTooLarge();
+        if (Number(response.headers.get("content-length")) > maxBytes)
+            throw new PinnedBodyTooLarge();
+        const chunks = [];
+        let total = 0;
+        if (response.body)
+            for await (const chunk of response.body) {
+                total += chunk.byteLength;
+                if (total > maxBytes)
+                    throw new PinnedBodyTooLarge();
+                chunks.push(Buffer.from(chunk));
+            }
+        return Buffer.concat(chunks, total); // Empty text remains a valid import input.
+    }
+    catch (error) {
+        throw error;
+    }
+    finally {
+        response.cancel();
+    }
+}

package/lib/promptImport/githubFolder.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { promptImportError } from "./errors.js";
 import { errInfo } from "../errInfo.js";
+import { publicPinnedHttpGet, readPinnedBody, PinnedBodyTooLarge } from "../pinnedHttpGet.js";
 const GITHUB_HOST = "github.com";
 const GITHUB_API_HOST = "api.github.com";
 const RAW_HOST = "raw.githubusercontent.com";
@@ -27,7 +28,13 @@ function safePath(path) {
     if (decoded.includes("\\") || decoded.split("/").includes("..")) {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub folder traversal is not allowed");
     }
-    return decoded.replace(/^\/+|\/+$/g, "");
+    let start = 0;
+    let end = decoded.length;
+    while (start < end && decoded[start] === "/")
+        start++;
+    while (end > start && decoded[end - 1] === "/")
+        end--;
+    return decoded.slice(start, end);
 }
 function assertOwnerRepo(owner, repo) {
     if (!OWNER_REPO_RE.test(owner || "") || !OWNER_REPO_RE.test(repo || "")) {
@@ -118,7 +125,7 @@ function assertGithubApiUrl(rawUrl) {
     catch {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub folder fetch returned an invalid URL");
     }
-    if (!["http:", "https:"].includes(url.protocol) || url.hostname !== GITHUB_API_HOST) {
+    if (url.protocol !== "https:" || url.hostname !== GITHUB_API_HOST) {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub folder fetch used an unsupported host");
     }
 }
@@ -130,7 +137,7 @@ function assertRawDownloadUrl(rawUrl) {
     catch {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub folder file has an invalid download URL");
     }
-    if (!["http:", "https:"].includes(url.protocol) || url.hostname !== RAW_HOST) {
+    if (url.protocol !== "https:" || url.hostname !== RAW_HOST) {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub folder file download host is unsupported");
     }
     const path = safePath(url.pathname.split("/").filter(Boolean).slice(3).join("/"));
@@ -177,20 +184,24 @@ async function fetchJson(url, limits) {
     assertGithubApiUrl(url);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), limits.fetchTimeoutMs);
+    let response;
     try {
-        const response = await fetch(url, {
-            signal: controller.signal,
-            headers: { Accept: "application/vnd.github+json" },
-        });
-        assertGithubApiUrl(response.url || url);
-        if (response.status === 404)
+        response = await publicPinnedHttpGet(url, controller.signal, assertGithubApiUrl, { Accept: "application/vnd.github+json", "User-Agent": "ima2-gen" });
+        if (response.status === 404) {
+            await response.cancel();
             return { notFound: true };
-        if (!response.ok) {
+        }
+        if (response.status < 200 || response.status >= 300) {
+            await response.cancel();
             throw promptImportError("GITHUB_FOLDER_NOT_FOUND", `GitHub folder fetch failed with ${response.status}`, 422);
         }
-        return { json: await response.json() };
+        const body = await readPinnedBody(response, limits.maxFileBytesForPreview);
+        return { json: JSON.parse(new TextDecoder().decode(body)) };
     }
     catch (error) {
+        if (error instanceof PinnedBodyTooLarge) {
+            throw promptImportError("REMOTE_FILE_TOO_LARGE", "GitHub folder index is too large", 413);
+        }
         const err = errInfo(error);
         if (err.name === "AbortError") {
             throw promptImportError("REMOTE_FETCH_TIMEOUT", "GitHub folder fetch timed out", 504);
@@ -198,6 +209,7 @@ async function fetchJson(url, limits) {
         throw error;
     }
     finally {
+        response?.cancel();
         clearTimeout(timer);
     }
 }
@@ -283,20 +295,18 @@ async function fetchRawFile(rawUrl, limits) {
     assertRawDownloadUrl(rawUrl);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), limits.fetchTimeoutMs);
+    let response;
     try {
-        const response = await fetch(rawUrl, { signal: controller.signal });
-        assertRawDownloadUrl(response.url || rawUrl);
-        if (!response.ok) {
+        response = await publicPinnedHttpGet(rawUrl, controller.signal, assertRawDownloadUrl);
+        if (response.status < 200 || response.status >= 300) {
+            await response.cancel();
             throw promptImportError("INVALID_GITHUB_SOURCE", `GitHub folder file fetch failed with ${response.status}`, 422);
         }
         const contentLength = Number(response.headers.get("content-length") || 0);
         if (contentLength > limits.maxFileBytesForPreview) {
             throw promptImportError("GITHUB_FOLDER_FILE_TOO_LARGE", "Folder file is too large", 413);
         }
-        const buffer = await response.arrayBuffer();
-        if (buffer.byteLength > limits.maxFileBytesForPreview) {
-            throw promptImportError("GITHUB_FOLDER_FILE_TOO_LARGE", "Folder file is too large", 413);
-        }
+        const buffer = await readPinnedBody(response, limits.maxFileBytesForPreview);
         const text = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
         return {
             text,
@@ -304,6 +314,9 @@ async function fetchRawFile(rawUrl, limits) {
         };
     }
     catch (error) {
+        if (error instanceof PinnedBodyTooLarge) {
+            throw promptImportError("GITHUB_FOLDER_FILE_TOO_LARGE", "Folder file is too large", 413);
+        }
         const err = errInfo(error);
         if (err.name === "AbortError") {
             throw promptImportError("REMOTE_FETCH_TIMEOUT", "GitHub folder file fetch timed out", 504);
@@ -311,6 +324,7 @@ async function fetchRawFile(rawUrl, limits) {
         throw error;
     }
     finally {
+        response?.cancel();
         clearTimeout(timer);
     }
 }

package/lib/promptImport/githubSource.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { promptImportError } from "./errors.js";
 import { errInfo } from "../errInfo.js";
+import { publicPinnedHttpGet, readPinnedBody, PinnedBodyTooLarge } from "../pinnedHttpGet.js";
 const ALLOWED_HOSTS = new Set(["github.com", "raw.githubusercontent.com"]);
 const SUPPORTED_EXTENSIONS = new Set(["md", "markdown", "txt"]);
 const OWNER_REPO_RE = /^[A-Za-z0-9_.-]+$/;
@@ -53,8 +54,8 @@ function normalizeUrlInput(input) {
     catch {
         return null;
     }
-    if (!["http:", "https:"].includes(url.protocol)) {
-        throw promptImportError("INVALID_GITHUB_SOURCE", "Only http(s) GitHub URLs are supported");
+    if (url.protocol !== "https:") {
+        throw promptImportError("INVALID_GITHUB_SOURCE", "Only HTTPS GitHub URLs are supported");
     }
     if (!ALLOWED_HOSTS.has(url.hostname)) {
         throw promptImportError("INVALID_GITHUB_SOURCE", "Only GitHub file URLs are supported");
@@ -157,7 +158,7 @@ function validateFinalFetchUrl(rawUrl) {
     catch {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub fetch returned an invalid final URL");
     }
-    if (!["http:", "https:"].includes(url.protocol)) {
+    if (url.protocol !== "https:") {
         throw promptImportError("INVALID_GITHUB_SOURCE", "GitHub fetch returned an unsupported protocol");
     }
     if (!ALLOWED_HOSTS.has(url.hostname)) {
@@ -189,20 +190,19 @@ export function normalizeGitHubSource(input) {
 export async function fetchGitHubSource(source, limits) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), limits.fetchTimeoutMs);
+    let response;
     try {
-        const response = await fetch(source.rawUrl, { signal: controller.signal });
-        validateFinalFetchUrl(response.url);
-        if (!response.ok) {
+        response = await publicPinnedHttpGet(source.rawUrl, controller.signal, validateFinalFetchUrl);
+        if (response.status < 200 || response.status >= 300) {
+            await response.cancel();
             throw promptImportError("INVALID_GITHUB_SOURCE", `GitHub file fetch failed with ${response.status}`, 422);
         }
         const contentLength = Number(response.headers.get("content-length") || 0);
         if (contentLength > limits.maxFileBytesForPreview) {
+            await response.cancel();
             throw promptImportError("REMOTE_FILE_TOO_LARGE", "Remote file is too large", 413);
         }
-        const buffer = await response.arrayBuffer();
-        if (buffer.byteLength > limits.maxFileBytesForPreview) {
-            throw promptImportError("REMOTE_FILE_TOO_LARGE", "Remote file is too large", 413);
-        }
+        const buffer = await readPinnedBody(response, limits.maxFileBytesForPreview);
         const text = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
         return {
             text,
@@ -213,6 +213,9 @@ export async function fetchGitHubSource(source, limits) {
         };
     }
     catch (error) {
+        if (error instanceof PinnedBodyTooLarge) {
+            throw promptImportError("REMOTE_FILE_TOO_LARGE", "Remote file is too large", 413);
+        }
         const err = errInfo(error);
         if (err.name === "AbortError") {
             throw promptImportError("REMOTE_FETCH_TIMEOUT", "GitHub fetch timed out", 504);
@@ -220,6 +223,7 @@ export async function fetchGitHubSource(source, limits) {
         throw error;
     }
     finally {
+        response?.cancel();
         clearTimeout(timer);
     }
 }

package/lib/promptImport/parsePromptCandidates.js

@@ -1,9 +1,39 @@
 import { createHash } from "crypto";
 function normalizeWhitespace(text) {
-    return text.replace(/\r\n/g, "\n").replace(/[ \t]+\n/g, "\n").trim();
+    return text.replace(/\r\n/g, "\n").split("\n").map((line, index, lines) => {
+        if (index === lines.length - 1)
+            return line;
+        let end = line.length;
+        while (end > 0 && (line[end - 1] === " " || line[end - 1] === "\t"))
+            end--;
+        return line.slice(0, end);
+    }).join("\n").trim();
 }
 function stripFrontmatter(text) {
-    return text.replace(/^---\s*\n[\s\S]*?\n---\s*\n/, "");
+    if (!text.startsWith("---"))
+        return text;
+    const openingEnd = text.indexOf("\n");
+    if (openingEnd < 0 || text.slice(3, openingEnd).trim() !== "")
+        return text;
+    let searchFrom = openingEnd + 1;
+    while (searchFrom < text.length) {
+        const closingStart = text.indexOf("\n---", searchFrom);
+        if (closingStart < 0)
+            break;
+        let end = closingStart + 4;
+        let lastNewline = -1;
+        // A complete delimiter requires whitespace through a newline. Consume the
+        // same trailing whitespace as the old regex without retrying suffixes.
+        while (end < text.length && /\s/.test(text[end])) {
+            if (text[end] === "\n")
+                lastNewline = end;
+            end++;
+        }
+        if (lastNewline >= 0)
+            return text.slice(lastNewline + 1);
+        searchFrom = end;
+    }
+    return text;
 }
 function isBoilerplate(line) {
     const trimmed = line.trim();