astro 4.13.3 → 4.14.0

package/dist/content/data-store.d.ts

@@ -0,0 +1,114 @@
+import { type PathLike } from 'fs';
+import type { MarkdownHeading } from '@astrojs/markdown-remark';
+export interface RenderedContent {
+    /** Rendered HTML string. If present then `render(entry)` will return a component that renders this HTML. */
+    html: string;
+    metadata?: {
+        /** Any images that are present in this entry. Relative to the {@link DataEntry} filePath. */
+        imagePaths?: Array<string>;
+        /** Any headings that are present in this file. */
+        headings?: MarkdownHeading[];
+        /** Raw frontmatter, parsed parsed from the file. This may include data from remark plugins. */
+        frontmatter?: Record<string, any>;
+        /** Any other metadata that is present in this file. */
+        [key: string]: unknown;
+    };
+}
+export interface DataEntry<TData extends Record<string, unknown> = Record<string, unknown>> {
+    /** The ID of the entry. Unique per collection. */
+    id: string;
+    /** The parsed entry data */
+    data: TData;
+    /** The file path of the content, if applicable. Relative to the site root. */
+    filePath?: string;
+    /** The raw body of the content, if applicable. */
+    body?: string;
+    /** An optional content digest, to check if the content has changed. */
+    digest?: number | string;
+    /** The rendered content of the entry, if applicable. */
+    rendered?: RenderedContent;
+    /**
+     * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase when calling `renderEntry`.
+     */
+    deferredRender?: boolean;
+}
+export declare class DataStore {
+    #private;
+    constructor();
+    get<T = DataEntry>(collectionName: string, key: string): T | undefined;
+    entries<T = DataEntry>(collectionName: string): Array<[id: string, T]>;
+    values<T = DataEntry>(collectionName: string): Array<T>;
+    keys(collectionName: string): Array<string>;
+    set(collectionName: string, key: string, value: unknown): void;
+    delete(collectionName: string, key: string): void;
+    clear(collectionName: string): void;
+    clearAll(): void;
+    has(collectionName: string, key: string): boolean;
+    hasCollection(collectionName: string): boolean;
+    collections(): Map<string, Map<string, any>>;
+    addAssetImport(assetImport: string, filePath: string): void;
+    addAssetImports(assets: Array<string>, filePath: string): void;
+    addModuleImport(fileName: string): void;
+    writeAssetImports(filePath: PathLike): Promise<void>;
+    writeModuleImports(filePath: PathLike): Promise<void>;
+    scopedStore(collectionName: string): ScopedDataStore;
+    /**
+     * Returns a MetaStore for a given collection, or if no collection is provided, the default meta collection.
+     */
+    metaStore(collectionName?: string): MetaStore;
+    toString(): string;
+    writeToDisk(filePath: PathLike): Promise<void>;
+    /**
+     * Attempts to load a DataStore from the virtual module.
+     * This only works in Vite.
+     */
+    static fromModule(): Promise<DataStore>;
+    static fromMap(data: Map<string, Map<string, any>>): Promise<DataStore>;
+    static fromString(data: string): Promise<DataStore>;
+    static fromFile(filePath: string | URL): Promise<DataStore>;
+}
+export interface ScopedDataStore {
+    get: <TData extends Record<string, unknown> = Record<string, unknown>>(key: string) => DataEntry<TData> | undefined;
+    entries: () => Array<[id: string, DataEntry]>;
+    set: <TData extends Record<string, unknown>>(opts: {
+        /** The ID of the entry. Must be unique per collection. */
+        id: string;
+        /** The data to store. */
+        data: TData;
+        /** The raw body of the content, if applicable. */
+        body?: string;
+        /** The file path of the content, if applicable. Relative to the site root. */
+        filePath?: string;
+        /** A content digest, to check if the content has changed. */
+        digest?: number | string;
+        /** The rendered content, if applicable. */
+        rendered?: RenderedContent;
+        /**
+         * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase.
+         */
+        deferredRender?: boolean;
+    }) => boolean;
+    values: () => Array<DataEntry>;
+    keys: () => Array<string>;
+    delete: (key: string) => void;
+    clear: () => void;
+    has: (key: string) => boolean;
+    /**
+     * Adds a single asset to the store. This asset will be transformed
+     * by Vite, and the URL will be available in the final build.
+     * @param fileName
+     * @param specifier
+     * @returns
+     */
+    addModuleImport: (fileName: string) => void;
+}
+/**
+ * A key-value store for metadata strings. Useful for storing things like sync tokens.
+ */
+export interface MetaStore {
+    get: (key: string) => string | undefined;
+    set: (key: string, value: string) => void;
+    has: (key: string) => boolean;
+    delete: (key: string) => void;
+}
+export declare function contentModuleToId(fileName: string): string;

package/dist/content/data-store.js

@@ -0,0 +1,323 @@
+import { promises as fs, existsSync } from "fs";
+import * as devalue from "devalue";
+import { imageSrcToImportId, importIdToSymbolName } from "../assets/utils/resolveImports.js";
+import { AstroError, AstroErrorData } from "../core/errors/index.js";
+import { CONTENT_MODULE_FLAG, DEFERRED_MODULE } from "./consts.js";
+const SAVE_DEBOUNCE_MS = 500;
+class DataStore {
+  #collections = /* @__PURE__ */ new Map();
+  #file;
+  #assetsFile;
+  #modulesFile;
+  #saveTimeout;
+  #assetsSaveTimeout;
+  #modulesSaveTimeout;
+  #dirty = false;
+  #assetsDirty = false;
+  #modulesDirty = false;
+  #assetImports = /* @__PURE__ */ new Set();
+  #moduleImports = /* @__PURE__ */ new Map();
+  constructor() {
+    this.#collections = /* @__PURE__ */ new Map();
+  }
+  get(collectionName, key) {
+    return this.#collections.get(collectionName)?.get(String(key));
+  }
+  entries(collectionName) {
+    const collection = this.#collections.get(collectionName) ?? /* @__PURE__ */ new Map();
+    return [...collection.entries()];
+  }
+  values(collectionName) {
+    const collection = this.#collections.get(collectionName) ?? /* @__PURE__ */ new Map();
+    return [...collection.values()];
+  }
+  keys(collectionName) {
+    const collection = this.#collections.get(collectionName) ?? /* @__PURE__ */ new Map();
+    return [...collection.keys()];
+  }
+  set(collectionName, key, value) {
+    const collection = this.#collections.get(collectionName) ?? /* @__PURE__ */ new Map();
+    collection.set(String(key), value);
+    this.#collections.set(collectionName, collection);
+    this.#saveToDiskDebounced();
+  }
+  delete(collectionName, key) {
+    const collection = this.#collections.get(collectionName);
+    if (collection) {
+      collection.delete(String(key));
+      this.#saveToDiskDebounced();
+    }
+  }
+  clear(collectionName) {
+    this.#collections.delete(collectionName);
+    this.#saveToDiskDebounced();
+  }
+  clearAll() {
+    this.#collections.clear();
+    this.#saveToDiskDebounced();
+  }
+  has(collectionName, key) {
+    const collection = this.#collections.get(collectionName);
+    if (collection) {
+      return collection.has(String(key));
+    }
+    return false;
+  }
+  hasCollection(collectionName) {
+    return this.#collections.has(collectionName);
+  }
+  collections() {
+    return this.#collections;
+  }
+  addAssetImport(assetImport, filePath) {
+    const id = imageSrcToImportId(assetImport, filePath);
+    if (id) {
+      this.#assetImports.add(id);
+      this.#writeAssetsImportsDebounced();
+    }
+  }
+  addAssetImports(assets, filePath) {
+    assets.forEach((asset) => this.addAssetImport(asset, filePath));
+  }
+  addModuleImport(fileName) {
+    const id = contentModuleToId(fileName);
+    if (id) {
+      this.#moduleImports.set(fileName, id);
+      this.#writeModulesImportsDebounced();
+    }
+  }
+  async writeAssetImports(filePath) {
+    this.#assetsFile = filePath;
+    if (this.#assetImports.size === 0) {
+      try {
+        await fs.writeFile(filePath, "export default new Map();");
+      } catch (err) {
+        throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err });
+      }
+    }
+    if (!this.#assetsDirty && existsSync(filePath)) {
+      return;
+    }
+    const imports = [];
+    const exports = [];
+    this.#assetImports.forEach((id) => {
+      const symbol = importIdToSymbolName(id);
+      imports.push(`import ${symbol} from '${id}';`);
+      exports.push(`[${JSON.stringify(id)}, ${symbol}]`);
+    });
+    const code = (
+      /* js */
+      `
+${imports.join("\n")}
+export default new Map([${exports.join(", ")}]);
+		`
+    );
+    try {
+      await fs.writeFile(filePath, code);
+    } catch (err) {
+      throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err });
+    }
+    this.#assetsDirty = false;
+  }
+  async writeModuleImports(filePath) {
+    this.#modulesFile = filePath;
+    if (this.#moduleImports.size === 0) {
+      try {
+        await fs.writeFile(filePath, "export default new Map();");
+      } catch (err) {
+        throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err });
+      }
+    }
+    if (!this.#modulesDirty && existsSync(filePath)) {
+      return;
+    }
+    const lines = [];
+    for (const [fileName, specifier] of this.#moduleImports) {
+      lines.push(`['${fileName}', () => import('${specifier}')]`);
+    }
+    const code = `
+export default new Map([
+${lines.join(",\n")}]);
+		`;
+    try {
+      await fs.writeFile(filePath, code);
+    } catch (err) {
+      throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err });
+    }
+    this.#modulesDirty = false;
+  }
+  #writeAssetsImportsDebounced() {
+    this.#assetsDirty = true;
+    if (this.#assetsFile) {
+      if (this.#assetsSaveTimeout) {
+        clearTimeout(this.#assetsSaveTimeout);
+      }
+      this.#assetsSaveTimeout = setTimeout(() => {
+        this.#assetsSaveTimeout = void 0;
+        this.writeAssetImports(this.#assetsFile);
+      }, SAVE_DEBOUNCE_MS);
+    }
+  }
+  #writeModulesImportsDebounced() {
+    this.#modulesDirty = true;
+    if (this.#modulesFile) {
+      if (this.#modulesSaveTimeout) {
+        clearTimeout(this.#modulesSaveTimeout);
+      }
+      this.#modulesSaveTimeout = setTimeout(() => {
+        this.#modulesSaveTimeout = void 0;
+        this.writeModuleImports(this.#modulesFile);
+      }, SAVE_DEBOUNCE_MS);
+    }
+  }
+  #saveToDiskDebounced() {
+    this.#dirty = true;
+    if (this.#file) {
+      if (this.#saveTimeout) {
+        clearTimeout(this.#saveTimeout);
+      }
+      this.#saveTimeout = setTimeout(() => {
+        this.#saveTimeout = void 0;
+        this.writeToDisk(this.#file);
+      }, SAVE_DEBOUNCE_MS);
+    }
+  }
+  scopedStore(collectionName) {
+    return {
+      get: (key) => this.get(collectionName, key),
+      entries: () => this.entries(collectionName),
+      values: () => this.values(collectionName),
+      keys: () => this.keys(collectionName),
+      set: ({ id: key, data, body, filePath, deferredRender, digest, rendered }) => {
+        if (!key) {
+          throw new Error(`ID must be a non-empty string`);
+        }
+        const id = String(key);
+        if (digest) {
+          const existing = this.get(collectionName, id);
+          if (existing && existing.digest === digest) {
+            return false;
+          }
+        }
+        const entry = {
+          id,
+          data
+        };
+        if (body) {
+          entry.body = body;
+        }
+        if (filePath) {
+          if (filePath.startsWith("/")) {
+            throw new Error(`File path must be relative to the site root. Got: ${filePath}`);
+          }
+          entry.filePath = filePath;
+        }
+        if (digest) {
+          entry.digest = digest;
+        }
+        if (rendered) {
+          entry.rendered = rendered;
+        }
+        if (deferredRender) {
+          entry.deferredRender = deferredRender;
+          if (filePath) {
+            this.addModuleImport(filePath);
+          }
+        }
+        this.set(collectionName, id, entry);
+        return true;
+      },
+      delete: (key) => this.delete(collectionName, key),
+      clear: () => this.clear(collectionName),
+      has: (key) => this.has(collectionName, key),
+      addAssetImport: (assetImport, fileName) => this.addAssetImport(assetImport, fileName),
+      addAssetImports: (assets, fileName) => this.addAssetImports(assets, fileName),
+      addModuleImport: (fileName) => this.addModuleImport(fileName)
+    };
+  }
+  /**
+   * Returns a MetaStore for a given collection, or if no collection is provided, the default meta collection.
+   */
+  metaStore(collectionName = ":meta") {
+    const collectionKey = `meta:${collectionName}`;
+    return {
+      get: (key) => this.get(collectionKey, key),
+      set: (key, data) => this.set(collectionKey, key, data),
+      delete: (key) => this.delete(collectionKey, key),
+      has: (key) => this.has(collectionKey, key)
+    };
+  }
+  toString() {
+    return devalue.stringify(this.#collections);
+  }
+  async writeToDisk(filePath) {
+    if (!this.#dirty) {
+      return;
+    }
+    try {
+      await fs.writeFile(filePath, this.toString());
+      this.#file = filePath;
+      this.#dirty = false;
+    } catch (err) {
+      throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err });
+    }
+  }
+  /**
+   * Attempts to load a DataStore from the virtual module.
+   * This only works in Vite.
+   */
+  static async fromModule() {
+    try {
+      const data = await import("astro:data-layer-content");
+      const map = devalue.unflatten(data.default);
+      return DataStore.fromMap(map);
+    } catch {
+    }
+    return new DataStore();
+  }
+  static async fromMap(data) {
+    const store = new DataStore();
+    store.#collections = data;
+    return store;
+  }
+  static async fromString(data) {
+    const map = devalue.parse(data);
+    return DataStore.fromMap(map);
+  }
+  static async fromFile(filePath) {
+    try {
+      if (existsSync(filePath)) {
+        const data = await fs.readFile(filePath, "utf-8");
+        return DataStore.fromString(data);
+      }
+    } catch {
+    }
+    return new DataStore();
+  }
+}
+function dataStoreSingleton() {
+  let instance = void 0;
+  return {
+    get: async () => {
+      if (!instance) {
+        instance = DataStore.fromModule();
+      }
+      return instance;
+    },
+    set: (store) => {
+      instance = store;
+    }
+  };
+}
+function contentModuleToId(fileName) {
+  const params = new URLSearchParams(DEFERRED_MODULE);
+  params.set("fileName", fileName);
+  params.set(CONTENT_MODULE_FLAG, "true");
+  return `${DEFERRED_MODULE}?${params.toString()}`;
+}
+const globalDataStore = dataStoreSingleton();
+export {
+  DataStore,
+  contentModuleToId,
+  globalDataStore
+};

package/dist/content/loaders/file.d.ts

@@ -0,0 +1,7 @@
+import type { Loader } from './types.js';
+/**
+ * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys.
+ * @todo Add support for other file types, such as YAML, CSV etc.
+ * @param fileName The path to the JSON file to load, relative to the content directory.
+ */
+export declare function file(fileName: string): Loader;

package/dist/content/loaders/file.js

@@ -0,0 +1,72 @@
+import { promises as fs, existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { posixRelative } from "../utils.js";
+function file(fileName) {
+  if (fileName.includes("*")) {
+    throw new Error("Glob patterns are not supported in `file` loader. Use `glob` loader instead.");
+  }
+  async function syncData(filePath, { logger, parseData, store, settings }) {
+    let json;
+    try {
+      const data = await fs.readFile(filePath, "utf-8");
+      json = JSON.parse(data);
+    } catch (error) {
+      logger.error(`Error reading data from ${fileName}`);
+      logger.debug(error.message);
+      return;
+    }
+    if (Array.isArray(json)) {
+      if (json.length === 0) {
+        logger.warn(`No items found in ${fileName}`);
+      }
+      logger.debug(`Found ${json.length} item array in ${fileName}`);
+      store.clear();
+      for (const rawItem of json) {
+        const id = (rawItem.id ?? rawItem.slug)?.toString();
+        if (!id) {
+          logger.error(`Item in ${fileName} is missing an id or slug field.`);
+          continue;
+        }
+        const data = await parseData({ id, data: rawItem, filePath });
+        store.set({
+          id,
+          data,
+          filePath: posixRelative(fileURLToPath(settings.config.root), filePath)
+        });
+      }
+    } else if (typeof json === "object") {
+      const entries = Object.entries(json);
+      logger.debug(`Found object with ${entries.length} entries in ${fileName}`);
+      store.clear();
+      for (const [id, rawItem] of entries) {
+        const data = await parseData({ id, data: rawItem, filePath });
+        store.set({ id, data });
+      }
+    } else {
+      logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
+    }
+  }
+  return {
+    name: "file-loader",
+    load: async (options) => {
+      const { settings, logger, watcher } = options;
+      logger.debug(`Loading data from ${fileName}`);
+      const url = new URL(fileName, settings.config.root);
+      if (!existsSync(url)) {
+        logger.error(`File not found: ${fileName}`);
+        return;
+      }
+      const filePath = fileURLToPath(url);
+      await syncData(filePath, options);
+      watcher?.on("change", async (changedPath) => {
+        if (changedPath === filePath) {
+          logger.info(`Reloading data from ${fileName}`);
+          await syncData(filePath, options);
+        }
+      });
+    }
+  };
+}
+export {
+  file
+};

package/dist/content/loaders/glob.d.ts

@@ -0,0 +1,25 @@
+import type { Loader } from './types.js';
+export interface GenerateIdOptions {
+    /** The path to the entry file, relative to the base directory. */
+    entry: string;
+    /** The base directory URL. */
+    base: URL;
+    /** The parsed, unvalidated data of the entry. */
+    data: Record<string, unknown>;
+}
+export interface GlobOptions {
+    /** The glob pattern to match files, relative to the base directory */
+    pattern: string;
+    /** The base directory to resolve the glob pattern from. Relative to the root directory, or an absolute file URL. Defaults to `.` */
+    base?: string | URL;
+    /**
+     * Function that generates an ID for an entry. Default implementation generates a slug from the entry path.
+     * @returns The ID of the entry. Must be unique per collection.
+     **/
+    generateId?: (options: GenerateIdOptions) => string;
+}
+/**
+ * Loads multiple entries, using a glob pattern to match files.
+ * @param pattern A glob pattern to match files, relative to the content directory.
+ */
+export declare function glob(globOptions: GlobOptions): Loader;