@agent-native/core 0.141.7 → 0.144.0

package/corpus/templates/slides/actions/import-file.ts

@@ -15,6 +15,7 @@ import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
 import { notifyClients } from "../server/handlers/decks.js";
+import { uploadPptxSlideImages } from "../server/handlers/import/pptx-assets.js";
 import { upsertBuilderProxyDesignSystem } from "../server/lib/builder-design-system-proxy.js";
 import { setupPdfParse } from "../server/lib/pdf-parse-setup.js";
 import {
@@ -32,7 +33,7 @@ export default defineAction({
     "Import a file (PPTX, DOCX, PDF, FIG) and extract content for creating slides or slide design systems. " +
     "For PPTX files, returns parsed slides with text and layout info ready for conversion. " +
     "For DOCX files, returns structured sections extracted from the document. " +
-    "For PDF files, returns extracted text organized by page. " +
+    "For PDF files, returns extracted text organized by page, or source-faithful page-image slides when importIntoDeck is true. " +
     "For Figma .fig files, requires Builder.io (free tier available) and starts Builder design-system indexing; the returned Builder job/design-system ids are the source of truth. " +
     "The agent can then use the extracted content to create a deck via create-deck or add-slide, or tell the user where Builder is indexing the design system.",
   schema: z.object({
@@ -53,7 +54,7 @@ export default defineAction({
       .optional()
       .default(false)
       .describe(
-        "If true, append slides converted from the file to the end of deckId's existing slides.",
+        "If true, append slides to deckId. PDF pages are imported as source-faithful images that preserve their original layout and aspect ratio; use the default false when editable extracted source text is needed.",
       ),
     maxChars: z.coerce
       .number()
@@ -176,6 +177,8 @@ export default defineAction({
           imageNames: slide.images.map((img) => img.name),
           notes: slide.notes,
           layoutHint: slide.layoutHint,
+          transition: slide.transition,
+          splitByParagraph: slide.splitByParagraph,
         })),
         theme: presentation.theme,
         deckId,
@@ -339,7 +342,7 @@ async function importPdfPagesAsFullBleedSlides(args: {
   canvasFactory: object;
 }) {
   const { fileBuffer, title, deckId, PDFParse, canvasFactory } = args;
-  const { buildFullBleedImageSlideHtml, convertSectionsToSlides } =
+  const { buildFullPageImageSlideHtml, convertSectionsToSlides } =
     await import("../server/handlers/import/html-converter.js");
 
   const pdf = new PDFParse({
@@ -347,31 +350,26 @@ async function importPdfPagesAsFullBleedSlides(args: {
     CanvasFactory: canvasFactory,
   });
   let pages: { num: number; text: string }[];
-  let imagesByPage: Map<
+  let screenshotsByPage: Map<
     number,
     { data: Uint8Array; width: number; height: number }
   >;
   try {
     pages = normalizePdfPages(await pdf.getText());
-    // Reuse the page's own embedded photo rather than rasterizing the whole
-    // page: headless canvas text rendering depends on the PDF's embedded
-    // fonts resolving in this runtime, which is unreliable, so real
-    // extracted text is drawn as HTML below instead of trusting the render.
-    const imageResult = await pdf.getImage({
+    // Rasterize the complete page so positioned text, vector shapes, logos,
+    // photos, and the source page proportions survive the import together.
+    const screenshotResult = await pdf.getScreenshot({
+      desiredWidth: 1600,
       imageBuffer: true,
       imageDataUrl: false,
     });
-    imagesByPage = new Map();
-    for (const page of imageResult.pages) {
-      const largest = page.images.reduce<
-        { data: Uint8Array; width: number; height: number } | undefined
-      >((best, image) => {
-        if (!best || image.width * image.height > best.width * best.height) {
-          return { data: image.data, width: image.width, height: image.height };
-        }
-        return best;
-      }, undefined);
-      if (largest) imagesByPage.set(page.pageNumber, largest);
+    screenshotsByPage = new Map();
+    for (const page of screenshotResult.pages) {
+      screenshotsByPage.set(page.pageNumber, {
+        data: page.data,
+        width: page.width,
+        height: page.height,
+      });
     }
   } finally {
     await pdf.destroy();
@@ -379,12 +377,12 @@ async function importPdfPagesAsFullBleedSlides(args: {
 
   // Source decks (e.g. Instagram carousel exports) are commonly portrait or
   // square, not the deck editor's 16:9 default — match the canvas to the
-  // first embedded photo's own proportions instead of stretching/cropping it.
-  const firstImage = pages
-    .map((page) => imagesByPage.get(page.num))
+  // first rendered page's own proportions instead of stretching/cropping it.
+  const firstScreenshot = pages
+    .map((page) => screenshotsByPage.get(page.num))
     .find((image): image is NonNullable<typeof image> => Boolean(image));
-  const aspectRatio = firstImage
-    ? nearestAspectRatio(firstImage.width, firstImage.height)
+  const aspectRatio = firstScreenshot
+    ? nearestAspectRatio(firstScreenshot.width, firstScreenshot.height)
     : undefined;
 
   const ownerEmail = getRequestUserEmail();
@@ -394,9 +392,9 @@ async function importPdfPagesAsFullBleedSlides(args: {
         .split(/\r?\n/)
         .map((line) => line.trim())
         .filter(Boolean);
-      const backgroundImage = imagesByPage.get(page.num);
+      const screenshot = screenshotsByPage.get(page.num);
 
-      if (!backgroundImage) {
+      if (!screenshot) {
         const [content] = convertSectionsToSlides([
           { heading: lines[0] ?? `Page ${page.num}`, content: page.text },
         ]);
@@ -409,7 +407,7 @@ async function importPdfPagesAsFullBleedSlides(args: {
       }
 
       const uploadResult = await uploadFile({
-        data: Buffer.from(backgroundImage.data),
+        data: Buffer.from(screenshot.data),
         filename: `slide-import-${Date.now()}-p${page.num}.png`,
         mimeType: "image/png",
         ownerEmail: ownerEmail ?? undefined,
@@ -420,22 +418,9 @@ async function importPdfPagesAsFullBleedSlides(args: {
           "File storage is not configured. Connect Builder.io (free tier available) or another upload provider before importing PDF slides.",
         );
       }
-      // pdf-parse breaks wrapped text across lines with no way to tell a
-      // wrapped title from a heading followed by a body paragraph. Treat a
-      // couple of short lines as one wrapped title (join them so the
-      // sentence isn't cut off); three or more lines as a heading plus body
-      // text, rendered with distinct sizes so they don't collapse into one
-      // flat paragraph.
-      const heading =
-        lines.length > 2 ? lines[0] : lines.join(" ") || undefined;
-      const subtitle = lines.length > 2 ? lines.slice(1).join(" ") : undefined;
       return {
         id: newSlideId(),
-        content: buildFullBleedImageSlideHtml(
-          uploadResult.url,
-          heading,
-          subtitle,
-        ),
+        content: buildFullPageImageSlideHtml(uploadResult.url),
         layout: "full-image",
         notes: page.text,
       };
@@ -459,65 +444,39 @@ function newSlideId(): string {
   return `slide-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
 }
 
-// EMF/WMF (Windows metafiles) and TIFF are valid PPTX embed formats but
-// browsers can't render them in an <img> tag — uploading and linking one
-// would just produce a broken image icon.
-const PPTX_BROWSER_RENDERABLE_IMAGE_MIME_TYPES = new Set([
-  "image/png",
-  "image/jpeg",
-  "image/gif",
-  "image/webp",
-  "image/svg+xml",
-  "image/bmp",
-]);
-
 async function buildPptxSlide(
   slide: import("../server/handlers/import/pptx-parser.js").ParsedSlide,
   slideIndex: number,
   ownerEmail: string,
   themeFont: string | undefined,
 ): Promise<{
-  slide: { id: string; content: string; layout: string; notes?: string };
+  slide: {
+    id: string;
+    content: string;
+    layout: string;
+    notes?: string;
+    transition?: "instant" | "none" | "fade" | "slide" | "zoom";
+    splitByParagraph?: boolean;
+  };
   imageSkippedCount: number;
 }> {
   const { convertToSlideHtml } =
     await import("../server/handlers/import/html-converter.js");
-  const image = slide.images[0];
-  const uploadable =
-    image && PPTX_BROWSER_RENDERABLE_IMAGE_MIME_TYPES.has(image.mimeType)
-      ? image
-      : undefined;
-  const imageUrl = uploadable
-    ? await uploadFile({
-        data: Buffer.from(uploadable.data),
-        filename:
-          "pptx-import-" +
-          Date.now() +
-          "-s" +
-          slideIndex +
-          "-" +
-          uploadable.name,
-        mimeType: uploadable.mimeType,
-        ownerEmail,
-        recordAsset: false,
-      })
-        .then((result) => result?.url)
-        // A single slide's upload failing (network/API/rate-limit)
-        // shouldn't abort the whole deck replacement — fall back to a
-        // placeholder like an unsupported format would.
-        .catch(() => undefined)
-    : undefined;
+  const uploadedImages = await uploadPptxSlideImages({
+    slide,
+    slideIndex,
+    ownerEmail,
+  });
   return {
     slide: {
       id: newSlideId(),
-      content: convertToSlideHtml(slide, imageUrl, themeFont),
+      content: convertToSlideHtml(slide, uploadedImages.urls, themeFont),
       layout: slide.layoutHint ?? "content",
       notes: slide.notes,
+      ...(slide.transition ? { transition: slide.transition } : {}),
+      ...(slide.splitByParagraph ? { splitByParagraph: true } : {}),
     },
-    // Only the first image on a slide is ever uploaded, so every other
-    // image on that slide is unconditionally dropped too — not just the
-    // first one when it's unsupported.
-    imageSkippedCount: Math.max(0, slide.images.length - (imageUrl ? 1 : 0)),
+    imageSkippedCount: uploadedImages.imageSkippedCount,
   };
 }
 
@@ -604,6 +563,8 @@ async function appendDeckSlides(
     content: string;
     layout: string;
     notes?: string;
+    transition?: "instant" | "none" | "fade" | "slide" | "zoom";
+    splitByParagraph?: boolean;
   }>,
   source: string,
   aspectRatio?: AspectRatio,

package/corpus/templates/slides/actions/import-google-slides-reference.ts

@@ -1,10 +1,19 @@
 import { defineAction } from "@agent-native/core";
+import { ssrfSafeFetch } from "@agent-native/core/extensions/url-safety";
 import { buildDeepLink } from "@agent-native/core/server";
 import { getRequestUserEmail } from "@agent-native/core/server/request-context";
+import pLimit from "p-limit";
 import { z } from "zod";
 
+import {
+  parsePptx,
+  type ParsedPresentation,
+} from "../server/handlers/import/pptx-parser.js";
 import { getGoogleDocsAccessToken } from "../server/lib/google-docs-oauth.js";
-import { importPptxBufferToDeck } from "./import-pptx.js";
+import {
+  importPptxBufferToDeck,
+  type ImportedImageFallback,
+} from "./import-pptx.js";
 
 function deckDeepLink(deckId: string): string {
   return buildDeepLink({
@@ -14,6 +23,38 @@ function deckDeepLink(deckId: string): string {
   });
 }
 
+export function extractGoogleSlidesPresentationId(value: string): string {
+  const candidate = value.trim();
+  if (!candidate)
+    throw new Error("A Google Slides file ID or URL is required.");
+
+  if (!/^https?:\/\//i.test(candidate)) {
+    if (!/^[a-zA-Z0-9_-]+$/.test(candidate)) {
+      throw new Error(
+        "Use a Google Slides file ID or a docs.google.com presentation URL.",
+      );
+    }
+    return candidate;
+  }
+
+  let url: URL;
+  try {
+    url = new URL(candidate);
+  } catch {
+    throw new Error("Use a valid Google Slides presentation URL.");
+  }
+  if (url.hostname !== "docs.google.com") {
+    throw new Error("Use a docs.google.com Google Slides presentation URL.");
+  }
+  const match = url.pathname.match(
+    /^\/presentation\/d\/([a-zA-Z0-9_-]+)(?:\/|$)/,
+  );
+  if (!match) {
+    throw new Error("That URL is not a Google Slides presentation link.");
+  }
+  return match[1];
+}
+
 async function exportGoogleSlidesAsPptx(fileId: string, accessToken: string) {
   const response = await fetch(
     `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=application/vnd.openxmlformats-officedocument.presentationml.presentation`,
@@ -36,36 +77,254 @@ async function exportGoogleSlidesAsPptx(fileId: string, accessToken: string) {
   return Buffer.from(bytes);
 }
 
+interface GoogleSlidesImageElement {
+  objectId?: string;
+  size?: {
+    width?: { magnitude?: number };
+    height?: { magnitude?: number };
+  };
+  transform?: {
+    scaleX?: number;
+    scaleY?: number;
+    translateX?: number;
+    translateY?: number;
+  };
+  image?: {
+    contentUrl?: string;
+    imageProperties?: {
+      cropProperties?: {
+        leftOffset?: number;
+        rightOffset?: number;
+        topOffset?: number;
+        bottomOffset?: number;
+      };
+    };
+  };
+}
+
+interface GoogleSlidesPresentationImages {
+  slides?: Array<{
+    pageElements?: GoogleSlidesImageElement[];
+  }>;
+}
+
+interface GoogleSlidesImageCandidate {
+  slideIndex: number;
+  imageIndex: number;
+  objectId?: string;
+  contentUrl: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  crop?: ImportedImageFallback["crop"];
+}
+
+function finiteNumber(value: number | undefined): number | undefined {
+  return value != null && Number.isFinite(value) ? value : undefined;
+}
+
+/**
+ * Google Drive's PPTX export can omit image page elements that remain present
+ * in the native Slides document. Read those native objects as a fidelity
+ * fallback so a direct Google Slides import does not silently lose artwork.
+ */
+async function fetchGoogleSlidesImageFallbacks(
+  fileId: string,
+  accessToken: string,
+  presentation: ParsedPresentation,
+): Promise<ImportedImageFallback[]> {
+  const fields =
+    "slides(pageElements(objectId,size,transform,image(contentUrl,imageProperties(cropProperties))))";
+  const response = await ssrfSafeFetch(
+    `https://slides.googleapis.com/v1/presentations/${encodeURIComponent(fileId)}?fields=${encodeURIComponent(fields)}`,
+    { headers: { Authorization: `Bearer ${accessToken}` } },
+    { httpsOnly: true, maxRedirects: 2 },
+  );
+  if (!response.ok) {
+    throw new Error(
+      `Google Slides returned HTTP ${response.status} while reading native image elements. Reconnect Google Drive and try the import again.`,
+    );
+  }
+  const googlePresentation =
+    (await response.json()) as GoogleSlidesPresentationImages;
+  const candidates: GoogleSlidesImageCandidate[] = [];
+  const candidateKeys = new Set<string>();
+
+  for (const [slideIndex, slide] of (
+    googlePresentation.slides ?? []
+  ).entries()) {
+    for (const [imageIndex, element] of (slide.pageElements ?? []).entries()) {
+      const contentUrl = element.image?.contentUrl;
+      const baseWidth = finiteNumber(element.size?.width?.magnitude);
+      const baseHeight = finiteNumber(element.size?.height?.magnitude);
+      const scaleX = finiteNumber(element.transform?.scaleX) ?? 1;
+      const scaleY = finiteNumber(element.transform?.scaleY) ?? 1;
+      const x = finiteNumber(element.transform?.translateX);
+      const y = finiteNumber(element.transform?.translateY);
+      const width =
+        baseWidth == null ? undefined : baseWidth * Math.abs(scaleX);
+      const height =
+        baseHeight == null ? undefined : baseHeight * Math.abs(scaleY);
+      if (
+        !contentUrl ||
+        x == null ||
+        y == null ||
+        width == null ||
+        height == null ||
+        width <= 0 ||
+        height <= 0
+      ) {
+        continue;
+      }
+
+      const existingImage = presentation.slides[slideIndex]?.elements.some(
+        (existing) =>
+          existing.kind === "image" &&
+          Math.abs(existing.x - x) < 1000 &&
+          Math.abs(existing.y - y) < 1000 &&
+          Math.abs(existing.width - width) < 1000 &&
+          Math.abs(existing.height - height) < 1000,
+      );
+      if (existingImage) continue;
+
+      const cropProperties = element.image?.imageProperties?.cropProperties;
+      const crop = cropProperties
+        ? {
+            left: cropProperties.leftOffset ?? 0,
+            top: cropProperties.topOffset ?? 0,
+            right: cropProperties.rightOffset ?? 0,
+            bottom: cropProperties.bottomOffset ?? 0,
+          }
+        : undefined;
+      const candidateKey = `${slideIndex}:${x}:${y}:${width}:${height}`;
+      if (candidateKeys.has(candidateKey)) continue;
+      candidateKeys.add(candidateKey);
+      candidates.push({
+        slideIndex,
+        imageIndex,
+        ...(element.objectId ? { objectId: element.objectId } : {}),
+        contentUrl,
+        x,
+        y,
+        width,
+        height,
+        ...(crop ? { crop } : {}),
+      });
+    }
+  }
+
+  const downloadLimit = pLimit(4);
+  const imageDownloads = new Map<
+    string,
+    Promise<{ data: Uint8Array; mimeType: string }>
+  >();
+  const downloadImage = (candidate: GoogleSlidesImageCandidate) => {
+    const existing = imageDownloads.get(candidate.contentUrl);
+    if (existing) return existing;
+
+    const download = downloadLimit(async () => {
+      const imageResponse = await ssrfSafeFetch(
+        candidate.contentUrl,
+        { headers: { Authorization: `Bearer ${accessToken}` } },
+        { httpsOnly: true, maxRedirects: 2 },
+      );
+      if (!imageResponse.ok) {
+        throw new Error(
+          `Google Slides returned HTTP ${imageResponse.status} while downloading image ${candidate.imageIndex + 1} on slide ${candidate.slideIndex + 1}.`,
+        );
+      }
+      const mimeType =
+        imageResponse.headers.get("content-type")?.split(";", 1)[0] ||
+        "image/png";
+      if (!mimeType.startsWith("image/")) {
+        throw new Error(
+          `Google Slides returned a non-image asset for image ${candidate.imageIndex + 1} on slide ${candidate.slideIndex + 1}.`,
+        );
+      }
+      return {
+        data: new Uint8Array(await imageResponse.arrayBuffer()),
+        mimeType,
+      };
+    });
+    imageDownloads.set(candidate.contentUrl, download);
+    return download;
+  };
+
+  return Promise.all(
+    candidates.map(async (candidate) => {
+      const { data, mimeType } = await downloadImage(candidate);
+      return {
+        slideIndex: candidate.slideIndex,
+        x: candidate.x,
+        y: candidate.y,
+        width: candidate.width,
+        height: candidate.height,
+        data,
+        mimeType,
+        name: `google-slides-${candidate.slideIndex + 1}-${candidate.objectId ?? candidate.imageIndex}.png`,
+        ...(candidate.crop ? { crop: candidate.crop } : {}),
+      };
+    }),
+  );
+}
+
 export default defineAction({
   description:
-    "Import a Google Slides deck selected from Google Picker and save it as a reusable Slides reference deck. " +
+    "Import a Google Slides deck selected from Google Picker or provided as a Google Slides URL and save it as a reusable Slides reference deck. " +
     "The selected presentation is exported from Drive as PPTX, parsed, and stored as a normal deck in the caller's Slides workspace.",
-  schema: z.object({
-    fileId: z.string().describe("Google Slides file ID from Google Picker"),
-    title: z
-      .string()
-      .optional()
-      .describe("Optional title for the imported reference deck"),
-  }),
-  run: async ({ fileId, title }) => {
+  schema: z
+    .object({
+      fileId: z
+        .string()
+        .optional()
+        .describe("Google Slides file ID from Google Picker"),
+      presentationUrl: z
+        .string()
+        .url()
+        .optional()
+        .describe("A docs.google.com Google Slides presentation URL"),
+      title: z
+        .string()
+        .optional()
+        .describe("Optional title for the imported reference deck"),
+    })
+    .refine(
+      ({ fileId, presentationUrl }) => Boolean(fileId || presentationUrl),
+      {
+        message: "Provide either fileId or presentationUrl.",
+      },
+    ),
+  run: async ({ fileId, presentationUrl, title }) => {
     const owner = getRequestUserEmail();
     if (!owner) throw new Error("no authenticated user");
 
     const connection = await getGoogleDocsAccessToken(owner);
     if (!connection) {
       throw new Error(
-        "Connect Google Docs before importing a Google Slides deck.",
+        "Google Drive is not connected. Use the Connect Google button in Slides, then try again.",
       );
     }
 
+    const presentationId = extractGoogleSlidesPresentationId(
+      presentationUrl ?? fileId ?? "",
+    );
     const fileBuffer = await exportGoogleSlidesAsPptx(
-      fileId,
+      presentationId,
+      connection.accessToken,
+    );
+    const parsedPresentation = await parsePptx(fileBuffer);
+    const imageFallbacks = await fetchGoogleSlidesImageFallbacks(
+      presentationId,
       connection.accessToken,
+      parsedPresentation,
     );
     return importPptxBufferToDeck({
       fileBuffer,
+      parsedPresentation,
       title,
       source: "import-google-slides-reference",
+      imageFallbacks,
     });
   },
   link: ({ result }) => {