astro-loader-pocketbase 0.1.0

package/README.md

@@ -0,0 +1,42 @@
+# astro-pocketbase-loader
+
+This package is a simple loader to load data from a PocketBase database into Astro using the [Astro Loader API](https://5-0-0-beta.docs.astro.build/en/reference/loader-reference/) introduced in Astro 5.
+
+## Usage
+
+In your content configuration file, you can use the `pocketbaseLoader` function to use your PocketBase database as a data source.
+
+```ts
+import { pocketbaseLoader } from "astro-pocketbase-loader";
+import { defineCollection } from "astro:content";
+
+const blog = defineCollection({
+  loader: pocketbaseLoader({
+    url: "https://<your-pocketbase-url>",
+    collectionName: "<collection-in-pocketbase>",
+    content: "<field-in-collection>"
+  })
+});
+```
+
+### Type Generation
+
+The loader can automatically generate types for your collection.
+To do this, you need to provide the email and password of an admin of the PocketBase instance.
+Under the hood, the loader will use the [PocketBase API](https://pocketbase.io/docs/api-collections/#view-collection) to fetch the schema of your collection and generate types with Zod based on that schema.
+
+### Private Collections
+
+If you want to access a private collection, you also need to provide the admin credentials.
+Otherwise, you need to make the collection public in the PocketBase dashboard.
+
+### Options
+
+| Option           | Type                      | Required | Description                                                                                                                         |
+| ---------------- | ------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------- |
+| `url`            | `string`                  | x        | The URL of your PocketBase instance.                                                                                                |
+| `collectionName` | `string`                  | x        | The name of the collection in your PocketBase instance.                                                                             |
+| `content`        | `string \| Array<string>` | x        | The field in the collection to use as content. This can also be an array of fields.                                                 |
+| `adminEmail`     | `string`                  |          | The email of the admin of the PocketBase instance. This is used for automatic type generation and access to private collections.    |
+| `adminPassword`  | `string`                  |          | The password of the admin of the PocketBase instance. This is used for automatic type generation and access to private collections. |
+| `forceUpdate`    | `boolean`                 |          | If set to `true`, the loader will fetch every entry instead of only the ones modified since the last build.                         |

package/index.ts

@@ -0,0 +1,5 @@
+import { pocketbaseLoader } from "./src/pocketbase-loader";
+import type { PocketBaseLoaderOptions } from "./src/types/pocketbase-loader-options.type";
+
+export { pocketbaseLoader };
+export type { PocketBaseLoaderOptions };

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "astro-loader-pocketbase",
+  "description": "A content loader for Astro that uses the PocketBase API",
+  "homepage": "https://github.com/pawcoding/astro-loader-pocketbase",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    ".": "./index.ts"
+  },
+  "files": [
+    "src",
+    "index.ts"
+  ],
+  "author": {
+    "name": "Luis Wolf",
+    "email": "development@pawcode.de",
+    "url": "https://pawcode.de"
+  },
+  "keywords": [
+    "withastro",
+    "astro",
+    "astro-loader",
+    "astro-content-loader",
+    "pocketbase"
+  ],
+  "scripts": {},
+  "devDependencies": {
+    "astro": "^5.0.0-beta.1"
+  },
+  "peerDependencies": {
+    "astro": "^5.0.0"
+  },
+  "dependencies": {
+    "@astrojs/check": "^0.9.3",
+    "typescript": "^5.6.2"
+  },
+  "license": "MIT"
+}

package/src/cleanup-entries.ts

@@ -0,0 +1,92 @@
+import type { LoaderContext } from "astro/loaders";
+import type { PocketBaseLoaderOptions } from "./types/pocketbase-loader-options.type";
+
+/**
+ * Cleanup entries that are no longer in the collection.
+ *
+ * @param options Options for the loader.
+ * @param context Context of the loader.
+ * @param adminToken Admin token to access all resources.
+ */
+export async function cleanupEntries(
+  options: PocketBaseLoaderOptions,
+  context: LoaderContext,
+  adminToken: string | undefined
+): Promise<void> {
+  // Build the URL for the collections endpoint
+  const collectionUrl = new URL(
+    `api/collections/${options.collectionName}/records`,
+    options.url
+  ).href;
+
+  // Create the headers for the request to append the admin token (if available)
+  const collectionHeaders = new Headers();
+  if (adminToken) {
+    collectionHeaders.set("Authorization", adminToken);
+  }
+
+  // Prepare pagination variables
+  let page = 1;
+  let totalPages = 0;
+  const entries = new Set<string>();
+
+  // Fetch all ids of the collection
+  do {
+    // Fetch ids from the collection
+    const collectionRequest = await fetch(
+      `${collectionUrl}?page=${page}&perPage=1000&fields=id`,
+      {
+        headers: collectionHeaders
+      }
+    );
+
+    // If the request was not successful, print the error message and return
+    if (!collectionRequest.ok) {
+      // If the collection is locked, an admin token is required
+      if (collectionRequest.status === 403) {
+        context.logger.error(
+          `The collection ${options.collectionName} is not accessible without an admin rights. Please provide an admin email and password in the config.`
+        );
+        return;
+      }
+
+      const reason = await collectionRequest
+        .json()
+        .then((data) => data.message);
+      const errorMessage = `Fetching ids from ${options.collectionName} failed with status code ${collectionRequest.status}.\nReason: ${reason}`;
+      context.logger.error(errorMessage);
+      return;
+    }
+
+    // Get the data from the response
+    const response = await collectionRequest.json();
+
+    // Add the ids to the set
+    for (const item of response.items) {
+      entries.add(item.id);
+    }
+
+    // Update the page and total pages
+    page = response.page;
+    totalPages = response.totalPages;
+  } while (page < totalPages);
+
+  let cleanedUp = 0;
+
+  // Get all ids of the entries in the store
+  const storedIds = context.store.keys();
+  for (const id of storedIds) {
+    // If the id is not in the entries set, remove the entry from the store
+    if (!entries.has(id)) {
+      context.store.delete(id);
+      cleanedUp++;
+    }
+  }
+
+  if (cleanedUp > 0) {
+    // Log the number of cleaned up entries
+    context.logger.info(
+      `Cleaned up ${cleanedUp} old entries for ${context.collection}`
+    );
+  }
+}

package/src/generate-schema.ts

@@ -0,0 +1,99 @@
+import type { ZodSchema } from "astro/zod";
+import { z } from "astro/zod";
+import type { PocketBaseLoaderOptions } from "./types/pocketbase-loader-options.type";
+import { getAdminToken } from "./utils/get-admin-token";
+import { parseSchema } from "./utils/parse-schema";
+
+/**
+ * Basic schema for every PocketBase collection.
+ */
+const BASIC_SCHEMA = {
+  id: z.string().length(15),
+  collectionId: z.string().length(15),
+  collectionName: z.string(),
+  created: z.date({ coerce: true }),
+  updated: z.date({ coerce: true })
+};
+
+/**
+ * Generate a schema for the collection based on the collection's schema in PocketBase.
+ * If no login credentials are provided, a {@link BASIC_SCHEMA} for every PocketBase collection is returned.
+ *
+ * @param options Options for the loader. See {@link PocketBaseLoaderOptions} for more details.
+ */
+export async function generateSchema(
+  options: PocketBaseLoaderOptions
+): Promise<ZodSchema> {
+  // TODO: Add support for local schema files
+
+  // If no admin email and password are provided, we can't get the schema from the API
+  if (!options.adminEmail || !options.adminPassword) {
+    console.warn(
+      "Make sure to add an admin email and password to the config to get automatic type generation."
+    );
+
+    // Return the basic schema
+    return z.object(BASIC_SCHEMA);
+  }
+
+  // Get the admin token
+  const token = await getAdminToken(
+    options.url,
+    options.adminEmail,
+    options.adminPassword
+  );
+
+  // If the token is invalid, return the basic schema
+  if (!token) {
+    return z.object(BASIC_SCHEMA);
+  }
+
+  // Build URL and headers for the schema request
+  const schemaUrl = new URL(
+    `api/collections/${options.collectionName}`,
+    options.url
+  ).href;
+  const schemaHeaders = new Headers();
+  schemaHeaders.set("Authorization", token);
+
+  // Fetch the schema
+  const schemaRequest = await fetch(schemaUrl, {
+    headers: schemaHeaders
+  });
+
+  // If the request was not successful, return the basic schema
+  if (!schemaRequest.ok) {
+    const reason = await schemaRequest.json().then((data) => data.message);
+    const errorMessage = `Fetching schema from ${options.collectionName} failed with status code ${schemaRequest.status}.\nReason: ${reason}`;
+    console.error(errorMessage);
+
+    return z.object(BASIC_SCHEMA);
+  }
+
+  // Get the schema from the response
+  const schema = await schemaRequest.json();
+
+  // Parse the schema
+  const fields = parseSchema(schema.schema);
+
+  // Check if the content field is present
+  if (typeof options.content === "string" && !fields[options.content]) {
+    console.error(
+      `The content field "${options.content}" is not present in the schema of the collection "${options.collectionName}".`
+    );
+  } else if (Array.isArray(options.content)) {
+    for (const field of options.content) {
+      if (!fields[field]) {
+        console.error(
+          `The content field "${field}" is not present in the schema of the collection "${options.collectionName}".`
+        );
+      }
+    }
+  }
+
+  // Combine the basic schema with the parsed fields
+  return z.object({
+    ...BASIC_SCHEMA,
+    ...fields
+  });
+}

package/src/load-entries.ts

@@ -0,0 +1,97 @@
+import type { LoaderContext } from "astro/loaders";
+import type { PocketBaseLoaderOptions } from "./types/pocketbase-loader-options.type";
+import { parseEntry } from "./utils/parse-entry";
+
+/**
+ * Load (modified) entries from a PocketBase collection.
+ *
+ * @param options Options for the loader.
+ * @param context Context of the loader.
+ * @param adminToken Admin token to access all resources.
+ * @param lastModified Date of the last fetch to only update changed entries.
+ */
+export async function loadEntries(
+  options: PocketBaseLoaderOptions,
+  context: LoaderContext,
+  adminToken: string | undefined,
+  lastModified: string | undefined
+): Promise<void> {
+  // Build the URL for the collections endpoint
+  const collectionUrl = new URL(
+    `api/collections/${options.collectionName}/records`,
+    options.url
+  ).href;
+
+  // Create the headers for the request to append the admin token (if available)
+  const collectionHeaders = new Headers();
+  if (adminToken) {
+    collectionHeaders.set("Authorization", adminToken);
+  }
+
+  // Log the fetching of the entries
+  context.logger.info(
+    `Fetching${lastModified ? " modified" : ""} data for ${
+      context.collection
+    } from ${collectionUrl}${
+      lastModified ? ` starting at ${lastModified}` : ""
+    }${adminToken ? " with admin token" : ""}`
+  );
+
+  // Prepare pagination variables
+  let page = 1;
+  let totalPages = 0;
+  let entries = 0;
+
+  // Fetch all (modified) entries
+  do {
+    // Fetch entries from the collection
+    // If `lastModified` is set, only fetch entries that have been modified since the last fetch
+    const collectionRequest = await fetch(
+      `${collectionUrl}?page=${page}&perPage=100&sort=-updated,id${
+        lastModified ? `&filter=(updated>"${lastModified}")` : ""
+      }`,
+      {
+        headers: collectionHeaders
+      }
+    );
+
+    // If the request was not successful, print the error message and return
+    if (!collectionRequest.ok) {
+      // If the collection is locked, an admin token is required
+      if (collectionRequest.status === 403) {
+        context.logger.error(
+          `The collection ${options.collectionName} is not accessible without an admin rights. Please provide an admin email and password in the config.`
+        );
+        return;
+      }
+
+      // Get the reason for the error
+      const reason = await collectionRequest
+        .json()
+        .then((data) => data.message);
+      const errorMessage = `Fetching data from ${options.collectionName} failed with status code ${collectionRequest.status}.\nReason: ${reason}`;
+      context.logger.error(errorMessage);
+      return;
+    }
+
+    // Get the data from the response
+    const response = await collectionRequest.json();
+
+    // Parse and store the entries
+    for (const entry of response.items) {
+      await parseEntry(entry, context, options.content);
+    }
+
+    // Update the page and total pages
+    page = response.page;
+    totalPages = response.totalPages;
+    entries += response.items.length;
+  } while (page < totalPages);
+
+  // Log the number of fetched entries
+  context.logger.info(
+    `Fetched ${entries}${lastModified ? " changed" : ""} entries for ${
+      context.collection
+    }`
+  );
+}

package/src/pocketbase-loader.ts

@@ -0,0 +1,55 @@
+import type { Loader, LoaderContext } from "astro/loaders";
+import { cleanupEntries } from "./cleanup-entries";
+import { generateSchema } from "./generate-schema";
+import { loadEntries } from "./load-entries";
+import type { PocketBaseLoaderOptions } from "./types/pocketbase-loader-options.type";
+import { getAdminToken } from "./utils/get-admin-token";
+
+/**
+ * Loader for collections stored in PocketBase.
+ *
+ * @param options Options for the loader. See {@link PocketBaseLoaderOptions} for more details.
+ */
+export function pocketbaseLoader(options: PocketBaseLoaderOptions): Loader {
+  return {
+    name: "pocketbase-loader",
+    load: async (context: LoaderContext): Promise<void> => {
+      // Get the date of the last fetch to only update changed entries.
+      // If `forceUpdate` is set to `true`, this will be `undefined` to fetch all entries again.
+      const lastModified = options.forceUpdate
+        ? undefined
+        : context.meta.get("last-modified");
+
+      // Clear the store if we want to fetch all entries again
+      if (options.forceUpdate) {
+        context.store.clear();
+      }
+
+      // Try to get an admin token to access all resources.
+      let token: string | undefined;
+      if (options.adminEmail && options.adminPassword) {
+        token = await getAdminToken(
+          options.url,
+          options.adminEmail,
+          options.adminPassword,
+          context.logger
+        );
+      }
+
+      if (context.store.keys().length > 0) {
+        // Cleanup entries that are no longer in the collection
+        await cleanupEntries(options, context, token);
+      }
+
+      // Load the (modified) entries
+      await loadEntries(options, context, token, lastModified);
+
+      // Set the last modified date to the current date
+      context.meta.set("last-modified", new Date().toISOString());
+    },
+    schema: async () => {
+      // Generate the schema for the collection according to the API
+      return await generateSchema(options);
+    }
+  };
+}

package/src/types/pocketbase-loader-options.type.ts

@@ -0,0 +1,38 @@
+/**
+ * Options for the PocketBase loader.
+ */
+export type PocketBaseLoaderOptions = {
+  /**
+   * URL of the PocketBase instance.
+   */
+  url: string;
+  /**
+   * Name of the collection in PocketBase.
+   */
+  collectionName: string;
+  /**
+   * Content of the collection in PocketBase.
+   * This must be the name of a field in the collection that contains the content.
+   * The content will be parsed as HTML and rendered to the page.
+   *
+   * If you want to render multiple fields as main content, you can pass an array of field names.
+   * The loader will concatenate the content of all fields in the order they are defined in the array.
+   * Each block will be contained in a `<section>` element.
+   */
+  content: string | Array<string>;
+  /**
+   * Email of an admin to get full access to the PocketBase instance.
+   * Together with `adminPassword` this is required to get automatic type generation and to access all resources even if they are not public.
+   */
+  adminEmail?: string;
+  /**
+   * Password of the admin to get full access to the PocketBase instance.
+   * Together with `adminEmail` this is required to get automatic type generation and to access all resources even if they are not public.
+   */
+  adminPassword?: string;
+  /**
+   * By default, the loader will only fetch entries that have been modified since the last fetch.
+   * If you want to fetch all entries, set this to `true`.
+   */
+  forceUpdate?: boolean;
+};

package/src/utils/get-admin-token.ts

@@ -0,0 +1,46 @@
+import type { AstroIntegrationLogger } from "astro";
+
+/**
+ * This function will get an admin token from the given PocketBase instance.
+ *
+ * @param url URL of the PocketBase instance
+ * @param email Email of the admin
+ * @param password Password of the admin
+ *
+ * @returns An admin token to access all resources of the PocketBase instance.
+ */
+export async function getAdminToken(
+  url: string,
+  email: string,
+  password: string,
+  logger?: AstroIntegrationLogger
+): Promise<string | undefined> {
+  // Build the URL for the login endpoint
+  const loginUrl = new URL(`api/admins/auth-with-password`, url).href;
+
+  // Create a new FormData object to send the login data
+  const loginData = new FormData();
+  loginData.set("identity", email);
+  loginData.set("password", password);
+
+  // Send the login request to get a token
+  const loginRequest = await fetch(loginUrl, {
+    method: "POST",
+    body: loginData
+  });
+
+  // If the login request was not successful, print the error message and return undefined
+  if (!loginRequest.ok) {
+    const reason = await loginRequest.json().then((data) => data.message);
+    const errorMessage = `The given email / password for ${url} was not correct. Astro can't generate type definitions automatically and may not have access to all resources.\nReason: ${reason}`;
+    if (logger) {
+      logger.error(errorMessage);
+    } else {
+      console.error(errorMessage);
+    }
+    return undefined;
+  }
+
+  // Return the token
+  return await loginRequest.json().then((data) => data.token);
+}

package/src/utils/parse-entry.ts

@@ -0,0 +1,49 @@
+import type { LoaderContext } from "astro/loaders";
+
+/**
+ * Parse an entry from PocketBase to match the schema and store it in the store.
+ *
+ * @param entry Entry to parse.
+ * @param context Context of the loader.
+ * @param contentFields Field(s) to use as content for the entry.
+ *                      If multiple fields are used, they will be concatenated and wrapped in `<section>` elements.
+ */
+export async function parseEntry(
+  entry: Record<string, any>,
+  { generateDigest, parseData, store }: LoaderContext,
+  contentFields: string | Array<string>
+): Promise<void> {
+  // Parse the data to match the schema
+  // This will throw an error if the data does not match the schema
+  const data = await parseData({
+    id: entry.id,
+    data: entry
+  });
+
+  // Generate a digest for the entry
+  // We can use the updated data as an identifier if the entry has changed since PocketBase automatically updates this value on every change
+  const digest = generateDigest(entry.updated);
+
+  // Generate the content for the entry
+  let content: string;
+  if (typeof contentFields === "string") {
+    // Only one field is used as content
+    content = entry[contentFields];
+  } else {
+    // Multiple fields are used as content, wrap each block in a section and concatenate them
+    content = contentFields
+      .map((field) => entry[field])
+      .map((block) => `<section>${block}</section>`)
+      .join("");
+  }
+
+  // Store the entry
+  store.set({
+    id: entry.id,
+    data,
+    digest,
+    rendered: {
+      html: content
+    }
+  });
+}

package/src/utils/parse-schema.ts

@@ -0,0 +1,81 @@
+import { z } from "astro/zod";
+
+export function parseSchema(
+  schema: Array<Record<string, any>>
+): Record<string, z.ZodType> {
+  // Prepare the schemas fields
+  const fields: Record<string, z.ZodType> = {};
+
+  // Parse every field in the schema
+  for (const field of schema) {
+    let fieldType;
+
+    // Determine the field type and create the corresponding Zod type
+    switch (field.type) {
+      case "number":
+        // Coerce the value to a number
+        fieldType = z.number({ coerce: true });
+        break;
+      case "bool":
+        // Coerce the value to a boolean
+        fieldType = z.boolean({ coerce: true });
+        break;
+      case "date":
+        // Coerce and parse the value as a date
+        fieldType = z.date({ coerce: true });
+        break;
+      case "select":
+        // Create an enum for the select values
+        const values = z.enum(field.options.values);
+
+        // Parse the field type based on the number of values it can have
+        fieldType = parseSingleOrMultipleValues(field as any, values);
+        break;
+      case "relation":
+      case "file":
+        // NOTE: Relations and files are currently not supported and are treated as strings
+
+        // Parse the field type based on the number of values it can have
+        fieldType = parseSingleOrMultipleValues(field as any, z.string());
+        break;
+      case "json":
+        // Parse the field as an object
+        fieldType = z.object({});
+        break;
+      default:
+        // Default to a string
+        fieldType = z.string();
+        break;
+    }
+
+    // If the field is not required, mark it as optional
+    if (!field.required) {
+      fieldType.isOptional();
+    }
+
+    // Add the field to the fields object
+    fields[field.name] = fieldType;
+  }
+
+  return fields;
+}
+
+/**
+ * Parse the field type based on the number of values it can have
+ *
+ * @param field Field to parse
+ * @param type Type of each value
+ *
+ * @returns The parsed field type
+ */
+function parseSingleOrMultipleValues(
+  field: { options: { maxSelect: number } },
+  type: z.ZodType
+) {
+  // If the select allows multiple values, create an array of the enum
+  if (field.options.maxSelect === 1) {
+    return type;
+  } else {
+    return z.array(type);
+  }
+}