astro 4.13.4 → 4.14.1

package/dist/content/loaders/file.d.ts

@@ -0,0 +1,7 @@
+import type { Loader } from './types.js';
+/**
+ * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys.
+ * @todo Add support for other file types, such as YAML, CSV etc.
+ * @param fileName The path to the JSON file to load, relative to the content directory.
+ */
+export declare function file(fileName: string): Loader;

package/dist/content/loaders/file.js

@@ -0,0 +1,72 @@
+import { promises as fs, existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { posixRelative } from "../utils.js";
+function file(fileName) {
+  if (fileName.includes("*")) {
+    throw new Error("Glob patterns are not supported in `file` loader. Use `glob` loader instead.");
+  }
+  async function syncData(filePath, { logger, parseData, store, settings }) {
+    let json;
+    try {
+      const data = await fs.readFile(filePath, "utf-8");
+      json = JSON.parse(data);
+    } catch (error) {
+      logger.error(`Error reading data from ${fileName}`);
+      logger.debug(error.message);
+      return;
+    }
+    if (Array.isArray(json)) {
+      if (json.length === 0) {
+        logger.warn(`No items found in ${fileName}`);
+      }
+      logger.debug(`Found ${json.length} item array in ${fileName}`);
+      store.clear();
+      for (const rawItem of json) {
+        const id = (rawItem.id ?? rawItem.slug)?.toString();
+        if (!id) {
+          logger.error(`Item in ${fileName} is missing an id or slug field.`);
+          continue;
+        }
+        const data = await parseData({ id, data: rawItem, filePath });
+        store.set({
+          id,
+          data,
+          filePath: posixRelative(fileURLToPath(settings.config.root), filePath)
+        });
+      }
+    } else if (typeof json === "object") {
+      const entries = Object.entries(json);
+      logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
+      store.clear();
+      for (const [id, rawItem] of entries) {
+        const data = await parseData({ id, data: rawItem, filePath });
+        store.set({ id, data });
+      }
+    } else {
+      logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
+    }
+  }
+  return {
+    name: "file-loader",
+    load: async (options) => {
+      const { settings, logger, watcher } = options;
+      logger.debug(`Loading data from ${fileName}`);
+      const url = new URL(fileName, settings.config.root);
+      if (!existsSync(url)) {
+        logger.error(`File not found: ${fileName}`);
+        return;
+      }
+      const filePath = fileURLToPath(url);
+      await syncData(filePath, options);
+      watcher?.on("change", async (changedPath) => {
+        if (changedPath === filePath) {
+          logger.info(`Reloading data from ${fileName}`);
+          await syncData(filePath, options);
+        }
+      });
+    }
+  };
+}
+export {
+  file
+};

package/dist/content/loaders/glob.d.ts

@@ -0,0 +1,25 @@
+import type { Loader } from './types.js';
+export interface GenerateIdOptions {
+    /** The path to the entry file, relative to the base directory. */
+    entry: string;
+    /** The base directory URL. */
+    base: URL;
+    /** The parsed, unvalidated data of the entry. */
+    data: Record<string, unknown>;
+}
+export interface GlobOptions {
+    /** The glob pattern to match files, relative to the base directory */
+    pattern: string;
+    /** The base directory to resolve the glob pattern from. Relative to the root directory, or an absolute file URL. Defaults to `.` */
+    base?: string | URL;
+    /**
+     * Function that generates an ID for an entry. Default implementation generates a slug from the entry path.
+     * @returns The ID of the entry. Must be unique per collection.
+     **/
+    generateId?: (options: GenerateIdOptions) => string;
+}
+/**
+ * Loads multiple entries, using a glob pattern to match files.
+ * @param pattern A glob pattern to match files, relative to the content directory.
+ */
+export declare function glob(globOptions: GlobOptions): Loader;

package/dist/content/loaders/glob.js

@@ -0,0 +1,218 @@
+import { promises as fs } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import fastGlob from "fast-glob";
+import { bold, green } from "kleur/colors";
+import micromatch from "micromatch";
+import pLimit from "p-limit";
+import { getContentEntryIdAndSlug, getEntryConfigByExtMap, posixRelative } from "../utils.js";
+function generateIdDefault({ entry, base, data }) {
+  if (data.slug) {
+    return data.slug;
+  }
+  const entryURL = new URL(entry, base);
+  const { slug } = getContentEntryIdAndSlug({
+    entry: entryURL,
+    contentDir: base,
+    collection: ""
+  });
+  return slug;
+}
+function glob(globOptions) {
+  if (globOptions.pattern.startsWith("../")) {
+    throw new Error(
+      "Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead."
+    );
+  }
+  if (globOptions.pattern.startsWith("/")) {
+    throw new Error(
+      "Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead."
+    );
+  }
+  const generateId = globOptions?.generateId ?? generateIdDefault;
+  const fileToIdMap = /* @__PURE__ */ new Map();
+  return {
+    name: "glob-loader",
+    load: async ({ settings, logger, watcher, parseData, store, generateDigest }) => {
+      const renderFunctionByContentType = /* @__PURE__ */ new WeakMap();
+      const untouchedEntries = new Set(store.keys());
+      async function syncData(entry, base, entryType) {
+        if (!entryType) {
+          logger.warn(`No entry type found for ${entry}`);
+          return;
+        }
+        const fileUrl = new URL(entry, base);
+        const contents = await fs.readFile(fileUrl, "utf-8").catch((err) => {
+          logger.error(`Error reading ${entry}: ${err.message}`);
+          return;
+        });
+        if (!contents) {
+          logger.warn(`No contents found for ${entry}`);
+          return;
+        }
+        const { body, data } = await entryType.getEntryInfo({
+          contents,
+          fileUrl
+        });
+        const id = generateId({ entry, base, data });
+        untouchedEntries.delete(id);
+        const existingEntry = store.get(id);
+        const digest = generateDigest(contents);
+        if (existingEntry && existingEntry.digest === digest && existingEntry.filePath) {
+          if (existingEntry.deferredRender) {
+            store.addModuleImport(existingEntry.filePath);
+          }
+          if (existingEntry.rendered?.metadata?.imagePaths?.length) {
+            store.addAssetImports(
+              existingEntry.rendered.metadata.imagePaths,
+              existingEntry.filePath
+            );
+          }
+          await parseData(existingEntry);
+          return;
+        }
+        const filePath = fileURLToPath(fileUrl);
+        const relativePath = posixRelative(fileURLToPath(settings.config.root), filePath);
+        const parsedData = await parseData({
+          id,
+          data,
+          filePath
+        });
+        if (entryType.getRenderFunction) {
+          let render = renderFunctionByContentType.get(entryType);
+          if (!render) {
+            render = await entryType.getRenderFunction(settings);
+            renderFunctionByContentType.set(entryType, render);
+          }
+          let rendered = void 0;
+          try {
+            rendered = await render?.({
+              id,
+              data: parsedData,
+              body,
+              filePath,
+              digest
+            });
+          } catch (error) {
+            logger.error(`Error rendering ${entry}: ${error.message}`);
+          }
+          store.set({
+            id,
+            data: parsedData,
+            body,
+            filePath: relativePath,
+            digest,
+            rendered
+          });
+          if (rendered?.metadata?.imagePaths?.length) {
+            store.addAssetImports(rendered.metadata.imagePaths, relativePath);
+          }
+        } else if ("contentModuleTypes" in entryType) {
+          store.set({
+            id,
+            data: parsedData,
+            body,
+            filePath: relativePath,
+            digest,
+            deferredRender: true
+          });
+        } else {
+          store.set({ id, data: parsedData, body, filePath: relativePath, digest });
+        }
+        fileToIdMap.set(filePath, id);
+      }
+      const entryConfigByExt = getEntryConfigByExtMap([
+        ...settings.contentEntryTypes,
+        ...settings.dataEntryTypes
+      ]);
+      const baseDir = globOptions.base ? new URL(globOptions.base, settings.config.root) : settings.config.root;
+      if (!baseDir.pathname.endsWith("/")) {
+        baseDir.pathname = `${baseDir.pathname}/`;
+      }
+      const files = await fastGlob(globOptions.pattern, {
+        cwd: fileURLToPath(baseDir)
+      });
+      function configForFile(file) {
+        const ext = file.split(".").at(-1);
+        if (!ext) {
+          logger.warn(`No extension found for ${file}`);
+          return;
+        }
+        return entryConfigByExt.get(`.${ext}`);
+      }
+      const limit = pLimit(10);
+      const skippedFiles = [];
+      const contentDir = new URL("content/", settings.config.srcDir);
+      function isInContentDir(file) {
+        const fileUrl = new URL(file, baseDir);
+        return fileUrl.href.startsWith(contentDir.href);
+      }
+      const configFiles = new Set(
+        ["config.js", "config.ts", "config.mjs"].map((file) => new URL(file, contentDir).href)
+      );
+      function isConfigFile(file) {
+        const fileUrl = new URL(file, baseDir);
+        return configFiles.has(fileUrl.href);
+      }
+      await Promise.all(
+        files.map((entry) => {
+          if (isConfigFile(entry)) {
+            return;
+          }
+          if (isInContentDir(entry)) {
+            skippedFiles.push(entry);
+            return;
+          }
+          return limit(async () => {
+            const entryType = configForFile(entry);
+            await syncData(entry, baseDir, entryType);
+          });
+        })
+      );
+      const skipCount = skippedFiles.length;
+      if (skipCount > 0) {
+        logger.warn(`The glob() loader cannot be used for files in ${bold("src/content")}.`);
+        if (skipCount > 10) {
+          logger.warn(
+            `Skipped ${green(skippedFiles.length)} files that matched ${green(globOptions.pattern)}.`
+          );
+        } else {
+          logger.warn(`Skipped the following files that matched ${green(globOptions.pattern)}:`);
+          skippedFiles.forEach((file) => logger.warn(`\u2022 ${green(file)}`));
+        }
+      }
+      untouchedEntries.forEach((id) => store.delete(id));
+      if (!watcher) {
+        return;
+      }
+      const matcher = micromatch.makeRe(globOptions.pattern);
+      const matchesGlob = (entry) => !entry.startsWith("../") && matcher.test(entry);
+      const basePath = fileURLToPath(baseDir);
+      async function onChange(changedPath) {
+        const entry = posixRelative(basePath, changedPath);
+        if (!matchesGlob(entry)) {
+          return;
+        }
+        const entryType = configForFile(changedPath);
+        const baseUrl = pathToFileURL(basePath);
+        await syncData(entry, baseUrl, entryType);
+        logger.info(`Reloaded data from ${green(entry)}`);
+      }
+      watcher.on("change", onChange);
+      watcher.on("add", onChange);
+      watcher.on("unlink", async (deletedPath) => {
+        const entry = posixRelative(basePath, deletedPath);
+        if (!matchesGlob(entry)) {
+          return;
+        }
+        const id = fileToIdMap.get(deletedPath);
+        if (id) {
+          store.delete(id);
+          fileToIdMap.delete(deletedPath);
+        }
+      });
+    }
+  };
+}
+export {
+  glob
+};

package/dist/content/loaders/index.d.ts

@@ -0,0 +1,3 @@
+export { file } from './file.js';
+export { glob } from './glob.js';
+export * from './types.js';

package/dist/content/loaders/index.js

@@ -0,0 +1,7 @@
+import { file } from "./file.js";
+import { glob } from "./glob.js";
+export * from "./types.js";
+export {
+  file,
+  glob
+};

package/dist/content/loaders/types.d.ts

@@ -0,0 +1,36 @@
+import type { FSWatcher } from 'vite';
+import type { ZodSchema } from 'zod';
+import type { AstroIntegrationLogger, AstroSettings } from '../../@types/astro.js';
+import type { MetaStore, ScopedDataStore } from '../mutable-data-store.js';
+export interface ParseDataOptions<TData extends Record<string, unknown>> {
+    /** The ID of the entry. Unique per collection */
+    id: string;
+    /** The raw, unvalidated data of the entry */
+    data: TData;
+    /** An optional file path, where the entry represents a local file. */
+    filePath?: string;
+}
+export interface LoaderContext {
+    /** The unique name of the collection */
+    collection: string;
+    /** A database abstraction to store the actual data */
+    store: ScopedDataStore;
+    /**  A simple KV store, designed for things like sync tokens */
+    meta: MetaStore;
+    logger: AstroIntegrationLogger;
+    settings: AstroSettings;
+    /** Validates and parses the data according to the collection schema */
+    parseData<TData extends Record<string, unknown>>(props: ParseDataOptions<TData>): Promise<TData>;
+    /** Generates a non-cryptographic content digest. This can be used to check if the data has changed */
+    generateDigest(data: Record<string, unknown> | string): string;
+    /** When running in dev, this is a filesystem watcher that can be used to trigger updates */
+    watcher?: FSWatcher;
+}
+export interface Loader {
+    /** Unique name of the loader, e.g. the npm package name */
+    name: string;
+    /** Do the actual loading of the data */
+    load: (context: LoaderContext) => Promise<void>;
+    /** Optionally, define the schema of the data. Will be overridden by user-defined schema */
+    schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
+}

package/dist/content/mutable-data-store.d.ts

@@ -0,0 +1,77 @@
+import { type PathLike } from 'node:fs';
+import { type DataEntry, DataStore, type RenderedContent } from './data-store.js';
+/**
+ * Extends the DataStore with the ability to change entries and write them to disk.
+ * This is kept as a separate class to avoid needing node builtins at runtime, when read-only access is all that is needed.
+ */
+export declare class MutableDataStore extends DataStore {
+    #private;
+    set(collectionName: string, key: string, value: unknown): void;
+    delete(collectionName: string, key: string): void;
+    clear(collectionName: string): void;
+    clearAll(): void;
+    addAssetImport(assetImport: string, filePath: string): void;
+    addAssetImports(assets: Array<string>, filePath: string): void;
+    addModuleImport(fileName: string): void;
+    writeAssetImports(filePath: PathLike): Promise<void>;
+    writeModuleImports(filePath: PathLike): Promise<void>;
+    scopedStore(collectionName: string): ScopedDataStore;
+    /**
+     * Returns a MetaStore for a given collection, or if no collection is provided, the default meta collection.
+     */
+    metaStore(collectionName?: string): MetaStore;
+    toString(): string;
+    writeToDisk(filePath: PathLike): Promise<void>;
+    /**
+     * Attempts to load a MutableDataStore from the virtual module.
+     * This only works in Vite.
+     */
+    static fromModule(): Promise<MutableDataStore>;
+    static fromMap(data: Map<string, Map<string, any>>): Promise<MutableDataStore>;
+    static fromString(data: string): Promise<MutableDataStore>;
+    static fromFile(filePath: string | URL): Promise<MutableDataStore>;
+}
+export interface ScopedDataStore {
+    get: <TData extends Record<string, unknown> = Record<string, unknown>>(key: string) => DataEntry<TData> | undefined;
+    entries: () => Array<[id: string, DataEntry]>;
+    set: <TData extends Record<string, unknown>>(opts: {
+        /** The ID of the entry. Must be unique per collection. */
+        id: string;
+        /** The data to store. */
+        data: TData;
+        /** The raw body of the content, if applicable. */
+        body?: string;
+        /** The file path of the content, if applicable. Relative to the site root. */
+        filePath?: string;
+        /** A content digest, to check if the content has changed. */
+        digest?: number | string;
+        /** The rendered content, if applicable. */
+        rendered?: RenderedContent;
+        /**
+         * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase.
+         */
+        deferredRender?: boolean;
+    }) => boolean;
+    values: () => Array<DataEntry>;
+    keys: () => Array<string>;
+    delete: (key: string) => void;
+    clear: () => void;
+    has: (key: string) => boolean;
+    /**
+     * Adds a single asset to the store. This asset will be transformed
+     * by Vite, and the URL will be available in the final build.
+     * @param fileName
+     * @param specifier
+     * @returns
+     */
+    addModuleImport: (fileName: string) => void;
+}
+/**
+ * A key-value store for metadata strings. Useful for storing things like sync tokens.
+ */
+export interface MetaStore {
+    get: (key: string) => string | undefined;
+    set: (key: string, value: string) => void;
+    has: (key: string) => boolean;
+    delete: (key: string) => void;
+}