@studiocms/devapps 0.1.0-beta.18 → 0.1.0-beta.19

package/dist/effects/WordPressAPI/utils.js

@@ -0,0 +1,239 @@
+import fs, { constants } from "node:fs";
+import { access, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { AstroError } from "astro/errors";
+import * as cheerio from "cheerio";
+import sanitizeHtml from "sanitize-html";
+import { Console, Context, Effect, Layer, genLogger } from "studiocms/effect";
+import TurndownService from "turndown";
+import {
+  APIEndpointConfig,
+  DownloadImageConfig,
+  DownloadPostImageConfig,
+  StringConfig
+} from "./configs.js";
+const _TurndownService = new TurndownService({
+  bulletListMarker: "-",
+  codeBlockStyle: "fenced",
+  emDelimiter: "*"
+});
+class WordPressAPIUtils extends Effect.Service()("WordPressAPIUtils", {
+  effect: genLogger("@studiocms/devapps/effects/WordPressAPI/utils.effect")(function* () {
+    const failedDownloads = /* @__PURE__ */ new Set();
+    const TDService = Effect.fn(
+      (fn) => Effect.try({
+        try: () => fn(_TurndownService),
+        catch: (cause) => new AstroError(
+          "Turndown Error",
+          `Failed to convert HTML to Markdown: ${cause.message}`
+        )
+      })
+    );
+    const turndown = genLogger("@studiocms/devapps/effects/WordPressAPI/utils.effect.turndown")(
+      function* () {
+        const { str } = yield* StringConfig;
+        return yield* TDService((TD) => TD.turndown(str));
+      }
+    );
+    const stripHtml = genLogger("@studiocms/devapps/effects/WordPressAPI/utils.effect.stripHtml")(
+      function* () {
+        const { str } = yield* StringConfig;
+        return yield* Effect.try(() => sanitizeHtml(str));
+      }
+    );
+    const loadHTML = Effect.fn(
+      (fn) => Effect.try({
+        try: () => fn(cheerio.load),
+        catch: (err) => new AstroError("Error loading content", err instanceof Error ? err.message : `${err}`)
+      })
+    );
+    const cleanUpHtml = genLogger(
+      "@studiocms/devapps/effects/WordPressAPI/utils.effect.cleanUpHtml"
+    )(function* () {
+      const { str } = yield* StringConfig;
+      const data = yield* loadHTML((fn) => fn(str));
+      const images = data("img");
+      for (const image of images) {
+        data(image).removeAttr("class").removeAttr("width").removeAttr("height").removeAttr("data-recalc-dims").removeAttr("sizes").removeAttr("srcset");
+      }
+      data(".wp-polls").html(
+        "<em>Polls have been temporarily removed while we migrate to a new platform.</em>"
+      );
+      data(".wp-polls.loading").remove();
+      return data.html();
+    });
+    const fetchAll = (url, page = 1, results = []) => genLogger("@studiocms/devapps/effects/WordPressAPI/utils.effect.fetchAll")(function* () {
+      url.searchParams.set("per_page", "100");
+      url.searchParams.set("page", String(page));
+      const res = yield* Effect.tryPromise({
+        try: () => fetch(url),
+        catch: (err) => new AstroError(
+          "Unknown Error while querying API",
+          err instanceof Error ? err.message : `${err}`
+        )
+      });
+      let data = yield* Effect.tryPromise({
+        try: () => res.json(),
+        catch: (err) => new AstroError(
+          "Unknown Error while reading API data",
+          err instanceof Error ? err.message : `${err}`
+        )
+      });
+      if (!Array.isArray(data)) {
+        if (typeof data === "object") {
+          data = Object.entries(data).map(([id, val]) => {
+            if (typeof val === "object") return { id, ...val };
+            return { id };
+          });
+        } else {
+          return yield* Effect.fail(
+            new AstroError(
+              "Expected WordPress API to return an array of items.",
+              `Received ${typeof data}:
+
+\`\`\`json
+${JSON.stringify(data, null, 2)}
+\`\`\``
+            )
+          );
+        }
+      }
+      results.push(...data);
+      const totalPages = Number.parseInt(res.headers.get("X-WP-TotalPages") || "1");
+      yield* Console.log("Fetched page", page, "of", totalPages);
+      if (page < totalPages) {
+        yield* Console.log("Fetching next page...");
+        return yield* fetchAll(url, page + 1, results);
+      }
+      return results;
+    });
+    const apiEndpoint = genLogger(
+      "@studiocms/devapps/effects/WordPressAPI/utils.effect.apiEndpoint"
+    )(function* () {
+      const { endpoint, type, path: path2 } = yield* APIEndpointConfig;
+      if (!endpoint) {
+        return yield* Effect.fail(
+          new AstroError(
+            "Missing `endpoint` argument.",
+            "Please pass a URL to your WordPress website as the `endpoint` option to the WordPress importer. Most commonly this looks something like `https://example.com/`"
+          )
+        );
+      }
+      let newEndpoint = endpoint;
+      if (!newEndpoint.endsWith("/")) newEndpoint += "/";
+      const apiBase = new URL(newEndpoint);
+      if (type === "settings") {
+        apiBase.pathname = "wp-json/";
+        return apiBase;
+      }
+      apiBase.pathname = `wp-json/wp/v2/${type}/${path2 ? `${path2}/` : ""}`;
+      return apiBase;
+    });
+    const downloadImage = genLogger(
+      "@studiocms/devapps/effects/WordPressAPI/utils.effect.downloadImage"
+    )(function* () {
+      const { destination, imageUrl } = yield* DownloadImageConfig;
+      const fileExists = yield* Effect.tryPromise({
+        try: () => access(destination, constants.F_OK).then(() => true),
+        catch: () => false
+      });
+      if (fileExists) {
+        yield* Console.error("File already exists:", destination);
+        return true;
+      }
+      const allowedExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"];
+      const ext = path.extname(destination.toString()).toLowerCase();
+      if (!allowedExtensions.includes(ext)) {
+        yield* Console.error("Invalid file extension:", ext);
+        return false;
+      }
+      const response = yield* Effect.tryPromise(() => fetch(imageUrl));
+      const contentType = response.headers.get("content-type");
+      if (!contentType?.startsWith("image/")) {
+        yield* Console.error("Invalid content type:", contentType);
+        return false;
+      }
+      const contentLength = response.headers.get("content-length");
+      const maxSize = 100 * 1024 * 1024;
+      if (contentLength && Number.parseInt(contentLength) > maxSize) {
+        yield* Console.error("File too large:", contentLength);
+        return false;
+      }
+      if (response.ok && response.body) {
+        const reader = response.body.getReader();
+        const chunks = [];
+        let done = false;
+        while (!done) {
+          const { done: readerDone, value } = yield* Effect.tryPromise(() => reader.read());
+          if (value) chunks.push(value);
+          done = readerDone;
+        }
+        const fileBuffer = Buffer.concat(chunks);
+        yield* Effect.tryPromise(() => writeFile(destination, fileBuffer, { flag: "wx" }));
+        yield* Console.log("Downloaded image:", imageUrl);
+        return true;
+      }
+      yield* Console.error("Failed to download image:", imageUrl);
+      return false;
+    });
+    const downloadPostImage = genLogger(
+      "@studiocms/devapps/effects/WordPressAPI/utils.effect.downloadPostImage"
+    )(function* () {
+      const { str: src, pathToFolder } = yield* DownloadPostImageConfig;
+      if (!src || !pathToFolder) return;
+      if (!fs.existsSync(pathToFolder)) {
+        fs.mkdirSync(pathToFolder, { recursive: true });
+      }
+      const baseName = path.basename(src);
+      const fileName = baseName.split("?")[0];
+      if (!fileName) {
+        yield* Console.error("Invalid image URL:", src);
+        return void 0;
+      }
+      const destinationFile = path.resolve(pathToFolder, fileName);
+      if (fs.existsSync(destinationFile)) {
+        yield* Console.log(`Post/Page image "${destinationFile}" already exists, skipping...`);
+        return fileName;
+      }
+      const imageDownloaded = yield* downloadImage.pipe(
+        DownloadImageConfig.makeProvide(src, destinationFile)
+      );
+      if (!imageDownloaded) failedDownloads.add(src);
+      return imageDownloaded ? fileName : void 0;
+    });
+    const downloadAndUpdateImages = genLogger(
+      "@studiocms/devapps/effects/WordPressAPI/utils.effect.downloadAndUpdateImages"
+    )(function* () {
+      const { str: html, pathToFolder } = yield* DownloadPostImageConfig;
+      const data = yield* loadHTML((fn) => fn(html));
+      const images = data("img");
+      for (const image of images) {
+        const src = data(image).attr("src");
+        if (src) {
+          const newSrc = yield* downloadPostImage.pipe(
+            DownloadPostImageConfig.makeProvide(src, pathToFolder)
+          );
+          if (newSrc) {
+            data(image).attr("src", newSrc);
+          } else {
+            data(image).attr("src", src);
+          }
+        }
+      }
+      return data.html();
+    });
+    return {
+      turndown,
+      stripHtml,
+      cleanUpHtml,
+      fetchAll,
+      apiEndpoint,
+      downloadPostImage,
+      downloadAndUpdateImages
+    };
+  })
+}) {
+}
+export {
+  WordPressAPIUtils
+};

package/dist/effects/wpImporter.d.ts

@@ -0,0 +1,13 @@
+import { Effect } from 'studiocms/effect';
+import { AstroAPIContextProvider } from './WordPressAPI/configs.js';
+import { WordPressAPI } from './WordPressAPI/importers.js';
+declare const WPImporter_base: Effect.Service.Class<WPImporter, "WPImporter", {
+    readonly dependencies: readonly [import("effect/Layer").Layer<WordPressAPI, never, never>];
+    readonly effect: Effect.Effect<{
+        runPostEvent: Effect.Effect<Response, boolean | Error | import("studiocms/sdk/effect/db").LibSQLDatabaseError | import("studiocms/sdk/errors").SDKCoreError | import("effect/ParseResult").ParseError, AstroAPIContextProvider>;
+    }, never, WordPressAPI>;
+}>;
+export declare class WPImporter extends WPImporter_base {
+    static Provide: <A, E, R>(self: Effect.Effect<A, E, R>) => Effect.Effect<A, E, Exclude<R, WPImporter>>;
+}
+export {};

package/dist/effects/wpImporter.js

@@ -0,0 +1,87 @@
+import { Console, Effect, genLogger } from "studiocms/effect";
+import {
+  AstroAPIContextProvider,
+  ImportEndpointConfig,
+  ImportPostsEndpointConfig
+} from "./WordPressAPI/configs.js";
+import { WordPressAPI } from "./WordPressAPI/importers.js";
+const createResponse = (status, statusText) => new Response(null, {
+  status,
+  statusText,
+  headers: {
+    "Content-Type": "application/json",
+    "Access-Control-Allow-Headers": "*"
+  }
+});
+const createErrorResponse = (statusText) => createResponse(400, statusText);
+class WPImporter extends Effect.Service()("WPImporter", {
+  dependencies: [WordPressAPI.Default],
+  effect: genLogger("@studiocms/devapps/effects/wpImporter.effect")(function* () {
+    const WPAPI = yield* WordPressAPI;
+    const parseFormData = (formData, name, type, optional = false) => Effect.gen(function* () {
+      const data = formData.get(name);
+      if (!optional && !data || data === null) {
+        throw yield* Effect.fail(new Error(`Missing required form field: ${name}`));
+      }
+      switch (type) {
+        case "string":
+          return data.toString();
+        case "boolean": {
+          const value = data.toString().toLowerCase();
+          return value === "true" || value === "1" || value === "yes";
+        }
+        default:
+          throw yield* Effect.fail(
+            new Error(`Unsupported type '${type}' for form field: ${name}`)
+          );
+      }
+    });
+    const runPostEvent = genLogger("@studiocms/devapps/effects/wpImporter.effect.runPostEvent")(
+      function* () {
+        const { context } = yield* AstroAPIContextProvider;
+        const formData = yield* Effect.tryPromise(() => context.request.formData());
+        const url = yield* parseFormData(formData, "url", "string");
+        const type = yield* parseFormData(formData, "type", "string");
+        const useBlogPlugin = yield* Effect.orElse(
+          parseFormData(formData, "useBlogPlugin", "boolean", true),
+          () => Effect.succeed(false)
+        );
+        if (!url || !type) {
+          return createErrorResponse("Bad Request");
+        }
+        yield* Console.log(
+          "Starting Import:",
+          url,
+          "\n Type:",
+          type,
+          "\n useBlogPlugin:",
+          useBlogPlugin
+        );
+        switch (type) {
+          case "pages":
+            yield* WPAPI.importPagesFromWPAPI.pipe(ImportEndpointConfig.makeProvide(url));
+            break;
+          case "posts":
+            yield* WPAPI.importPostsFromWPAPI.pipe(
+              ImportPostsEndpointConfig.makeProvide(url, useBlogPlugin)
+            );
+            break;
+          case "settings":
+            yield* WPAPI.importSettingsFromWPAPI.pipe(ImportEndpointConfig.makeProvide(url));
+            break;
+          default:
+            return createErrorResponse("Bad Request: Invalid import type");
+        }
+        return createResponse(200, "success");
+      }
+    );
+    return {
+      runPostEvent
+    };
+  })
+}) {
+  static Provide = Effect.provide(this.Default);
+}
+export {
+  WPImporter
+};

package/dist/routes/wp-importer.d.ts

@@ -1,27 +1,2 @@
 import type { APIRoute } from 'astro';
-/**
- * Handles the POST request for importing data from a WordPress site.
- *
- * @param {APIContext} context - The context of the API request.
- * @param {Request} context.request - The incoming request object.
- *
- * @returns {Promise<Response>} The response object indicating the result of the import operation.
- *
- * The function expects the request to contain form data with the following fields:
- * - `url`: The URL of the WordPress site to import data from.
- * - `type`: The type of data to import (e.g., 'pages', 'posts', 'settings').
- * - `useBlogPlugin` (optional): A boolean value indicating whether to use the blog plugin for importing posts.
- *
- * The function performs the following steps:
- * 1. Extracts the form data from the request.
- * 2. Validates the presence and types of the `url` and `type` fields.
- * 3. Logs the import operation details.
- * 4. Based on the `type` field, calls the appropriate import function:
- *    - `importPagesFromWPAPI` for importing pages.
- *    - `importPostsFromWPAPI` for importing posts, optionally using the blog plugin.
- *    - `importSettingsFromWPAPI` for importing settings.
- * 5. Returns a response indicating success or failure.
- *
- * @throws {Error} If the `type` field contains an invalid value.
- */
 export declare const POST: APIRoute;

package/dist/routes/wp-importer.js

@@ -1,57 +1,12 @@
-import {
-  importPagesFromWPAPI,
-  importPostsFromWPAPI,
-  importSettingsFromWPAPI
-} from "../utils/wp-api/index.js";
-const POST = async ({ request }) => {
-  const data = await request.formData();
-  const url = data.get("url");
-  const type = data.get("type");
-  const useBlogPlugin = data.get("useBlogPlugin");
-  if (!url || !type) {
-    return new Response(null, {
-      status: 400,
-      statusText: "Bad Request",
-      headers: {
-        "Content-Type": "application/json",
-        "Access-Control-Allow-Headers": "*"
-      }
-    });
-  }
-  if (typeof url !== "string" || typeof type !== "string") {
-    return new Response(null, {
-      status: 400,
-      statusText: "Bad Request",
-      headers: {
-        "Content-Type": "application/json",
-        "Access-Control-Allow-Headers": "*"
-      }
-    });
-  }
-  console.log("Starting Import:", url, "\n Type:", type, "\n useBlogPlugin:", useBlogPlugin);
-  const useBlogPluginValue = useBlogPlugin === "true";
-  switch (type) {
-    case "pages":
-      await importPagesFromWPAPI(url);
-      break;
-    case "posts":
-      await importPostsFromWPAPI(url, useBlogPluginValue);
-      break;
-    case "settings":
-      await importSettingsFromWPAPI(url);
-      break;
-    default:
-      throw new Error("Invalid import type");
-  }
-  return new Response(null, {
-    status: 200,
-    statusText: "success",
-    headers: {
-      "Content-Type": "application/json",
-      "Access-Control-Allow-Headers": "*"
-    }
-  });
-};
+import { convertToVanilla, genLogger } from "studiocms/effect";
+import { AstroAPIContextProvider } from "../effects/WordPressAPI/configs.js";
+import { WPImporter } from "../effects/wpImporter.js";
+const POST = async (context) => await convertToVanilla(
+  genLogger("@studiocms/devapps/routes/wp-importer.POST")(function* () {
+    const WP = yield* WPImporter;
+    return yield* WP.runPostEvent.pipe(AstroAPIContextProvider.makeProvide(context));
+  }).pipe(WPImporter.Provide)
+);
 export {
   POST
 };

package/dist/virt.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="@astrojs/db" />
+/// <reference types="studiocms/v/types" />
 
 declare module 'virtual:studiocms-devapps/endpoints' {
 	export const wpAPIEndpoint: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@studiocms/devapps",
-  "version": "0.1.0-beta.18",
+  "version": "0.1.0-beta.19",
   "description": "A dedicated CMS for Astro DB. Built from the ground up by the Astro community.",
   "author": {
     "name": "Adam Matthiesen | Jacob Jenkins | Paul Valladares",
@@ -57,18 +57,18 @@
     "astro-integration-kit": "^0.18",
     "cheerio": "^1.0.0",
     "turndown": "^7.2.0",
-    "html-entities": "^2.5.2"
+    "sanitize-html": "^2.17.0"
   },
   "devDependencies": {
-    "@types/cheerio": "^0.22.35",
     "@types/turndown": "^5.0.5",
-    "typescript": "^5.7"
+    "@types/sanitize-html": "^2.16.0",
+    "typescript": "^5.8.2"
   },
   "peerDependencies": {
-    "@astrojs/db": "^0.14.10",
-    "astro": "^5.7.1",
-    "vite": "^6.2.6",
-    "studiocms": "0.1.0-beta.18"
+    "@astrojs/db": "^0.15",
+    "astro": "^5.9",
+    "vite": "^6.3.4",
+    "studiocms": "0.1.0-beta.19"
   },
   "peerDependenciesMeta": {
     "studiocms": {