@studiocms/devapps 0.1.0-beta.9 → 0.1.0

package/dist/utils/wp-api/converters.js

@@ -1,181 +0,0 @@
-import path from "node:path";
-import { db, eq } from "astro:db";
-import { userProjectRoot } from "virtual:studiocms-devapps/config";
-import { decode } from "html-entities";
-import { tsPageDataCategories, tsPageDataTags } from "studiocms/sdk/tables";
-import TurndownService from "turndown";
-import {
-  apiEndpoint,
-  cleanUpHtml,
-  downloadAndUpdateImages,
-  downloadPostImage,
-  stripHtml
-} from "./utils";
-const ASTROPUBLICFOLDER = path.resolve(userProjectRoot, "public");
-const WPImportFolder = path.resolve(ASTROPUBLICFOLDER, "wp-import");
-const pagesImagesFolder = path.resolve(WPImportFolder, "pages");
-const postsImagesFolder = path.resolve(WPImportFolder, "posts");
-const ConvertToPageData = async (page, endpoint) => {
-  const data = page;
-  const titleImageId = data.featured_media;
-  const titleImageURL = apiEndpoint(endpoint, "media", `${titleImageId}`);
-  const titleImageResponse = await fetch(titleImageURL);
-  const titleImageJson = await titleImageResponse.json();
-  const titleImage = await downloadPostImage(titleImageJson.source_url, pagesImagesFolder);
-  const pageData = {
-    id: crypto.randomUUID(),
-    title: data.title.rendered,
-    description: decode(stripHtml(data.excerpt.rendered)),
-    slug: data.slug,
-    publishedAt: new Date(data.date_gmt),
-    updatedAt: new Date(data.modified_gmt),
-    showOnNav: false,
-    contentLang: "default",
-    package: "studiocms"
-  };
-  if (titleImage) {
-    pageData.heroImage = titleImage;
-  }
-  return pageData;
-};
-const ConvertToPageContent = async (pageData, page) => {
-  const data = page;
-  if (pageData.id === void 0) {
-    throw new Error("pageData is missing id");
-  }
-  const cleanupContent = cleanUpHtml(data.content.rendered);
-  const htmlWithImages = await downloadAndUpdateImages(cleanupContent, pagesImagesFolder);
-  const turndownService = new TurndownService({
-    bulletListMarker: "-",
-    codeBlockStyle: "fenced",
-    emDelimiter: "*"
-  });
-  const content = turndownService.turndown(htmlWithImages);
-  const pageContent = {
-    id: crypto.randomUUID(),
-    contentId: pageData.id,
-    contentLang: "default",
-    content
-  };
-  return pageContent;
-};
-const generateCategories = async (categories, endpoint) => {
-  const newCategories = [];
-  for (const categoryId of categories) {
-    const categoryExists = await db.select().from(tsPageDataCategories).where(eq(tsPageDataCategories.id, categoryId)).get();
-    if (categoryExists) {
-      console.log(`Category with id ${categoryId} already exists in the database`);
-      continue;
-    }
-    const categoryURL = apiEndpoint(endpoint, "categories", `${categoryId}`);
-    const response = await fetch(categoryURL);
-    const json = await response.json();
-    newCategories.push(json);
-  }
-  if (newCategories.length > 0) {
-    const categoryData = newCategories.map((category) => {
-      const data = {
-        id: category.id,
-        name: category.name,
-        slug: category.slug,
-        description: category.description,
-        meta: JSON.stringify(category.meta)
-      };
-      if (category.parent) {
-        data.parent = category.parent;
-      }
-      return data;
-    });
-    for (const category of categoryData) {
-      console.log(`Inserting category with id ${category.id} into the database`);
-      await db.insert(tsPageDataCategories).values(category);
-    }
-  }
-};
-const generateTags = async (tags, endpoint) => {
-  const newTags = [];
-  for (const tagId of tags) {
-    const tagExists = await db.select().from(tsPageDataTags).where(eq(tsPageDataTags.id, tagId)).get();
-    if (tagExists) {
-      console.log(`Tag with id ${tagId} already exists in the database`);
-      continue;
-    }
-    const tagURL = apiEndpoint(endpoint, "tags", `${tagId}`);
-    const response = await fetch(tagURL);
-    const json = await response.json();
-    newTags.push(json);
-  }
-  if (newTags.length > 0) {
-    const tagData = newTags.map((tag) => {
-      const data = {
-        id: tag.id,
-        name: tag.name,
-        slug: tag.slug,
-        description: tag.description,
-        meta: JSON.stringify(tag.meta)
-      };
-      return data;
-    });
-    for (const tag of tagData) {
-      console.log(`Inserting tag with id ${tag.id} into the database`);
-      await db.insert(tsPageDataTags).values(tag);
-    }
-  }
-};
-const ConvertToPostData = async (post, useBlogPkg, endpoint) => {
-  const data = post;
-  const titleImageId = data.featured_media;
-  const titleImageURL = apiEndpoint(endpoint, "media", `${titleImageId}`);
-  const titleImageResponse = await fetch(titleImageURL);
-  const titleImageJson = await titleImageResponse.json();
-  const titleImage = await downloadPostImage(titleImageJson.source_url, pagesImagesFolder);
-  const pkg = useBlogPkg ? "@studiocms/blog" : "studiocms/markdown";
-  await generateCategories(data.categories, endpoint);
-  await generateTags(data.tags, endpoint);
-  const pageData = {
-    id: crypto.randomUUID(),
-    title: data.title.rendered,
-    description: decode(stripHtml(data.excerpt.rendered)),
-    slug: data.slug,
-    publishedAt: new Date(data.date_gmt),
-    updatedAt: new Date(data.modified_gmt),
-    showOnNav: false,
-    contentLang: "default",
-    package: pkg,
-    categories: JSON.stringify(data.categories),
-    tags: JSON.stringify(data.tags)
-  };
-  if (titleImage) {
-    pageData.heroImage = titleImage;
-  }
-  return pageData;
-};
-const ConvertToPostContent = async (pageData, post) => {
-  const data = post;
-  if (pageData.id === void 0) {
-    throw new Error("pageData is missing id");
-  }
-  const cleanupContent = cleanUpHtml(data.content.rendered);
-  const htmlWithImages = await downloadAndUpdateImages(cleanupContent, postsImagesFolder);
-  const turndownService = new TurndownService({
-    bulletListMarker: "-",
-    codeBlockStyle: "fenced",
-    emDelimiter: "*"
-  });
-  const content = turndownService.turndown(htmlWithImages);
-  const pageContent = {
-    id: crypto.randomUUID(),
-    contentId: pageData.id,
-    contentLang: "default",
-    content
-  };
-  return pageContent;
-};
-export {
-  ConvertToPageContent,
-  ConvertToPageData,
-  ConvertToPostContent,
-  ConvertToPostData,
-  generateCategories,
-  generateTags
-};

package/dist/utils/wp-api/index.d.ts

@@ -1,6 +0,0 @@
-import type { tsPageContent, tsPageData } from 'studiocms/sdk/tables';
-export type PageData = typeof tsPageData.$inferInsert;
-export type PageContent = typeof tsPageContent.$inferInsert;
-export * from './pages.js';
-export * from './posts.js';
-export * from './settings.js';

package/dist/utils/wp-api/index.js

@@ -1,3 +0,0 @@
-export * from "./pages.js";
-export * from "./posts.js";
-export * from "./settings.js";

package/dist/utils/wp-api/pages.d.ts

@@ -1,13 +0,0 @@
-/**
- * Imports pages from a WordPress API endpoint.
- *
- * This function fetches all pages from the specified WordPress API endpoint
- * and imports each page individually.
- *
- * @param endpoint - The WordPress API endpoint to fetch pages from.
- *
- * @returns A promise that resolves when all pages have been imported.
- *
- * @throws Will throw an error if the pages cannot be imported.
- */
-export declare const importPagesFromWPAPI: (endpoint: string) => Promise<void>;

package/dist/utils/wp-api/pages.js

@@ -1,38 +0,0 @@
-import { db } from "astro:db";
-import { tsPageContent, tsPageData } from "studiocms/sdk/tables";
-import { ConvertToPageContent, ConvertToPageData } from "./converters.js";
-import { apiEndpoint, fetchAll } from "./utils.js";
-const generatePageFromData = async (page, endpoint) => {
-  const pageData = await ConvertToPageData(page, endpoint);
-  const pageContent = await ConvertToPageContent(pageData, page);
-  return { pageData, pageContent };
-};
-const importPage = async (page, endpoint) => {
-  const { pageData, pageContent } = await generatePageFromData(page, endpoint);
-  const pageDataResult = await db.insert(tsPageData).values(pageData).returning({ id: tsPageData.id, title: tsPageData.title }).get();
-  if (pageDataResult === void 0) {
-    throw new Error("Failed to insert page data");
-  }
-  const pageContentResult = await db.insert(tsPageContent).values(pageContent).returning({ id: tsPageContent.id }).get();
-  if (pageContentResult === void 0) {
-    throw new Error("Failed to insert page content");
-  }
-  console.log("- Imported new page from WP-API: ", pageDataResult.title);
-};
-const importPagesFromWPAPI = async (endpoint) => {
-  const url = apiEndpoint(endpoint, "pages");
-  console.log("fetching pages from: ", url.origin);
-  const pages = await fetchAll(url);
-  console.log("Total pages: ", pages.length);
-  try {
-    for (const page of pages) {
-      console.log("importing page:", page.title.rendered);
-      await importPage(page, endpoint);
-    }
-  } catch (error) {
-    console.error("Failed to import pages from WP-API:", error);
-  }
-};
-export {
-  importPagesFromWPAPI
-};

package/dist/utils/wp-api/posts.d.ts

@@ -1,10 +0,0 @@
-/**
- * Imports posts from a WordPress API endpoint.
- *
- * @param endpoint - The API endpoint to fetch posts from.
- * @param useBlogPkg - A boolean indicating whether to use the blog package.
- * @returns A promise that resolves when all posts have been imported.
- *
- * @throws Will throw an error if the import process fails.
- */
-export declare const importPostsFromWPAPI: (endpoint: string, useBlogPkg: boolean) => Promise<void>;

package/dist/utils/wp-api/posts.js

@@ -1,38 +0,0 @@
-import { db } from "astro:db";
-import { tsPageContent, tsPageData } from "studiocms/sdk/tables";
-import { ConvertToPostContent, ConvertToPostData } from "./converters.js";
-import { apiEndpoint, fetchAll } from "./utils.js";
-const generatePostFromData = async (post, useBlogPkg, endpoint) => {
-  const pageData = await ConvertToPostData(post, useBlogPkg, endpoint);
-  const pageContent = await ConvertToPostContent(pageData, post);
-  return { pageData, pageContent };
-};
-const importPost = async (post, useBlogPkg, endpoint) => {
-  const { pageData, pageContent } = await generatePostFromData(post, useBlogPkg, endpoint);
-  const pageDataResult = await db.insert(tsPageData).values(pageData).returning({ id: tsPageData.id, title: tsPageData.title }).get();
-  if (pageDataResult === void 0) {
-    throw new Error("Failed to insert post data");
-  }
-  const pageContentResult = await db.insert(tsPageContent).values(pageContent).returning({ id: tsPageContent.id }).get();
-  if (pageContentResult === void 0) {
-    throw new Error("Failed to insert post content");
-  }
-  console.log("- Imported new post from WP-API:", pageDataResult.title);
-};
-const importPostsFromWPAPI = async (endpoint, useBlogPkg) => {
-  const url = apiEndpoint(endpoint, "posts");
-  console.log("Fetching posts from: ", url.origin);
-  const posts = await fetchAll(url);
-  console.log("Total posts: ", posts.length);
-  try {
-    for (const post of posts) {
-      console.log("importing post: ", post.title.rendered);
-      await importPost(post, useBlogPkg, endpoint);
-    }
-  } catch (error) {
-    console.error("Failed to import posts from WP-API: ", error);
-  }
-};
-export {
-  importPostsFromWPAPI
-};

package/dist/utils/wp-api/settings.d.ts

@@ -1,18 +0,0 @@
-/**
- * Imports site settings from a WordPress API endpoint and updates the local database.
- *
- * @param endpoint - The WordPress API endpoint to fetch settings from.
- *
- * This function performs the following steps:
- * 1. Constructs the URL for the settings endpoint.
- * 2. Fetches the site settings from the constructed URL.
- * 3. Logs the fetched settings.
- * 4. Downloads the site icon if available.
- * 5. If the site icon is not available, attempts to download the site logo.
- * 6. Constructs the site configuration object.
- * 7. Updates the local database with the fetched settings.
- * 8. Logs the success or failure of the database update.
- *
- * @throws Will log an error message if the fetch or database update fails.
- */
-export declare const importSettingsFromWPAPI: (endpoint: string) => Promise<void>;

package/dist/utils/wp-api/settings.js

@@ -1,45 +0,0 @@
-import path from "node:path";
-import { db, eq } from "astro:db";
-import { userProjectRoot } from "virtual:studiocms-devapps/config";
-import { CMSSiteConfigId } from "studiocms/consts";
-import { tsSiteConfig } from "studiocms/sdk/tables";
-import { apiEndpoint, downloadPostImage } from "./utils.js";
-const ASTROPUBLICFOLDER = path.resolve(userProjectRoot, "public");
-const importSettingsFromWPAPI = async (endpoint) => {
-  const url = apiEndpoint(endpoint, "settings");
-  console.log("Fetching site settings from: ", url.origin);
-  const response = await fetch(url);
-  const settings = await response.json();
-  console.log("Importing site settings: ", settings);
-  let siteIcon = void 0;
-  if (settings.site_icon_url) {
-    siteIcon = await downloadPostImage(settings.site_icon_url, ASTROPUBLICFOLDER);
-  }
-  if (!settings.site_icon_url && settings.site_logo) {
-    const siteLogoURL = apiEndpoint(endpoint, "media", `${settings.site_logo}`);
-    const siteLogoResponse = await fetch(siteLogoURL);
-    const siteLogoJson = await siteLogoResponse.json();
-    siteIcon = await downloadPostImage(siteLogoJson.source_url, ASTROPUBLICFOLDER);
-  }
-  const siteConfig = {
-    id: CMSSiteConfigId,
-    title: settings.name,
-    description: settings.description
-  };
-  if (siteIcon) {
-    siteConfig.siteIcon = siteIcon;
-  }
-  try {
-    const insert = await db.update(tsSiteConfig).set(siteConfig).where(eq(tsSiteConfig.id, CMSSiteConfigId)).returning({ id: tsSiteConfig.id }).get();
-    if (insert) {
-      console.log("Updated site settings");
-    } else {
-      console.error("Failed to update site settings");
-    }
-  } catch (error) {
-    console.error("Failed to import site settings from WP-API: ", error);
-  }
-};
-export {
-  importSettingsFromWPAPI
-};

package/dist/utils/wp-api/utils.d.ts

@@ -1,66 +0,0 @@
-/**
- * Removes all HTML tags from a given string.
- *
- * @param string - The input string containing HTML tags.
- * @returns The input string with all HTML tags removed.
- */
-export declare function stripHtml(string: string): string;
-/**
- * Cleans up the provided HTML string by removing certain attributes from images
- * and modifying specific elements related to WordPress polls.
- *
- * @param html - The HTML string to be cleaned up.
- * @returns The cleaned-up HTML string.
- */
-export declare const cleanUpHtml: (html: string) => string;
-/**
- * Downloads an image from the specified URL and saves it to the given destination.
- *
- * @param {string | URL} imageUrl - The URL of the image to download.
- * @param {string | URL} destination - The file path where the image should be saved.
- * @returns {Promise<boolean>} - A promise that resolves to true if the image was successfully downloaded,
- *                               or false if the download failed or the file already exists.
- *
- * @throws {Error} - Throws an error if there is an issue with the fetch request or file writing.
- */
-export declare function downloadImage(imageUrl: string | URL, destination: string | URL): Promise<boolean>;
-/**
- * Downloads an image from the given source URL and saves it to the specified folder.
- *
- * @param src - The URL of the image to download.
- * @param pathToFolder - The path to the folder where the image should be saved.
- * @returns The file name of the downloaded image if successful, otherwise `undefined`.
- *
- * @remarks
- * - If the `src` or `pathToFolder` parameters are not provided, the function will return immediately.
- * - If the specified folder does not exist, it will be created recursively.
- * - If the image already exists in the specified folder, the function will log a message and skip the download.
- * - If the image download fails, the source URL will be added to the `imagesNotDownloaded` array.
- */
-export declare const downloadPostImage: (src: string, pathToFolder: string) => Promise<string | undefined>;
-/**
- * Downloads and updates the image sources in the provided HTML string.
- *
- * This function parses the given HTML string, finds all image elements,
- * downloads the images to the specified folder, and updates the image
- * sources to point to the downloaded images.
- *
- * @param html - The HTML string containing image elements to be processed.
- * @param pathToFolder - The path to the folder where images should be downloaded.
- * @returns A promise that resolves to the updated HTML string with new image sources.
- */
-export declare const downloadAndUpdateImages: (html: string, pathToFolder: string) => Promise<string>;
-/**
- * Constructs a WordPress API endpoint URL based on the provided parameters.
- *
- * @param endpoint - The base URL of the WordPress website.
- * @param type - The type of resource to access. Can be 'posts', 'pages', 'media', 'categories', 'tags', or 'settings'.
- * @param path - An optional path to append to the endpoint.
- * @returns The constructed URL object pointing to the desired API endpoint.
- * @throws {AstroError} If the `endpoint` argument is missing.
- */
-export declare const apiEndpoint: (endpoint: string, type: "posts" | "pages" | "media" | "categories" | "tags" | "settings", path?: string) => URL;
-/**
- * Fetch all pages for a paginated WP endpoint.
- */
-export declare function fetchAll(url: URL, page?: number, results?: any[]): Promise<any[]>;

package/dist/utils/wp-api/utils.js

@@ -1,138 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-import { AstroError } from "astro/errors";
-import * as cheerio from "cheerio";
-const imagesNotDownloaded = [];
-function stripHtml(string) {
-  return string.replace(/<[^>]*>/g, "");
-}
-const cleanUpHtml = (html) => {
-  const $ = cheerio.load(html);
-  const images = $("img");
-  for (const image of images) {
-    $(image).removeAttr("class width height data-recalc-dims sizes srcset");
-  }
-  $(".wp-polls").html(
-    "<em>Polls have been temporarily removed while we migrate to a new platform.</em>"
-  );
-  $(".wp-polls-loading").remove();
-  return $.html();
-};
-async function downloadImage(imageUrl, destination) {
-  if (fs.existsSync(destination)) {
-    console.error("File already exists:", destination);
-    return true;
-  }
-  try {
-    const response = await fetch(imageUrl);
-    if (response.ok && response.body) {
-      const reader = response.body.getReader();
-      const chunks = [];
-      let done = false;
-      while (!done) {
-        const { done: readerDone, value } = await reader.read();
-        if (value) {
-          chunks.push(value);
-        }
-        done = readerDone;
-      }
-      const fileBuffer = Buffer.concat(chunks);
-      fs.writeFileSync(destination, fileBuffer, { flag: "wx" });
-      console.log("Downloaded image:", imageUrl);
-      return true;
-    }
-    console.error("Failed to download image:", imageUrl);
-    return false;
-  } catch (error) {
-    console.error("Failed to download image:", imageUrl, error);
-    return false;
-  }
-}
-const downloadPostImage = async (src, pathToFolder) => {
-  if (!src || !pathToFolder) {
-    return;
-  }
-  if (!fs.existsSync(pathToFolder)) {
-    fs.mkdirSync(pathToFolder, { recursive: true });
-  }
-  const fileName = path.basename(src).split("?")[0];
-  const destinationFile = path.resolve(pathToFolder, fileName);
-  if (fs.existsSync(destinationFile)) {
-    console.log(`Post/Page image "${destinationFile}" already exists, skipping...`);
-    return fileName;
-  }
-  const imageDownloaded = await downloadImage(src, destinationFile);
-  if (!imageDownloaded) {
-    imagesNotDownloaded.push(src);
-  }
-  return imageDownloaded ? fileName : void 0;
-};
-const downloadAndUpdateImages = async (html, pathToFolder) => {
-  const $ = cheerio.load(html);
-  const images = $("img");
-  for (const image of images) {
-    const src = $(image).attr("src");
-    if (src) {
-      const newSrc = await downloadPostImage(src, pathToFolder);
-      $(image).attr("src", newSrc);
-    }
-  }
-  return $.html();
-};
-const apiEndpoint = (endpoint, type, path2) => {
-  if (!endpoint) {
-    throw new AstroError(
-      "Missing `endpoint` argument.",
-      "Please pass a URL to your WordPress website as the `endpoint` option to the WordPress importer. Most commonly this looks something like `https://example.com/`"
-    );
-  }
-  let newEndpoint = endpoint;
-  if (!newEndpoint.endsWith("/")) newEndpoint += "/";
-  const apiBase = new URL(newEndpoint);
-  if (type === "settings") {
-    apiBase.pathname = "wp-json/";
-    return apiBase;
-  }
-  apiBase.pathname = `wp-json/wp/v2/${type}/${path2 ? `${path2}/` : ""}`;
-  return apiBase;
-};
-async function fetchAll(url, page = 1, results = []) {
-  url.searchParams.set("per_page", "100");
-  url.searchParams.set("page", String(page));
-  const response = await fetch(url);
-  let data = await response.json();
-  if (!Array.isArray(data)) {
-    if (typeof data === "object") {
-      data = Object.entries(data).map(([id, val]) => {
-        if (typeof val === "object") return { id, ...val };
-        return { id };
-      });
-    } else {
-      throw new AstroError(
-        "Expected WordPress API to return an array of items.",
-        `Received ${typeof data}:
-
-\`\`\`json
-${JSON.stringify(data, null, 2)}
-\`\`\``
-      );
-    }
-  }
-  results.push(...data);
-  const totalPages = Number.parseInt(response.headers.get("X-WP-TotalPages") || "1");
-  console.log("Fetched page", page, "of", totalPages);
-  if (page < totalPages) {
-    console.log("Fetching next page...");
-    return fetchAll(url, page + 1, results);
-  }
-  return results;
-}
-export {
-  apiEndpoint,
-  cleanUpHtml,
-  downloadAndUpdateImages,
-  downloadImage,
-  downloadPostImage,
-  fetchAll,
-  stripHtml
-};