@content-collections/core 0.1.0

package/src/collector.test.ts

@@ -0,0 +1,157 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { createCollector } from "./collector";
+import { Events, createEmitter } from "./events";
+
+describe("collector", () => {
+  let emitter = createEmitter<Events>();
+
+  beforeEach(() => {
+    emitter = createEmitter<Events>();
+  });
+
+  describe("collectFile", () => {
+    it("should collect file", async () => {
+      const { collectFile } = createCollector(emitter);
+
+      const file = await collectFile(
+        __dirname,
+        "./__tests__/sources/test/001.md"
+      );
+
+      if (!file) {
+        throw new Error("File not found");
+      }
+
+      expect(file.path).toBe("./__tests__/sources/test/001.md");
+      expect(file.data.content.trim()).toBe("# One");
+      expect(file.data.name).toBe("One");
+    });
+
+    it("should throw an error if file does not exist", async () => {
+      emitter.on("collector:read-error", ({ error }) => {
+        throw error;
+      });
+      const { collectFile } = createCollector(emitter);
+
+      await expect(
+        collectFile(__dirname, "./__tests__/sources/test/notfound.md")
+      ).rejects.toThrow("no such file or directory");
+    });
+
+    it("should capture the error and return null", async () => {
+      emitter.on("collector:read-error", ({ error }) => {
+        expect(error.type).toBe("Read");
+        expect(error.message).toMatch(/no such file or directory/);
+      });
+
+      const { collectFile } = createCollector(emitter, __dirname);
+      const file = await collectFile(
+        __dirname,
+        "./__tests__/sources/test/notfound.md"
+      );
+      expect(file).toBeNull();
+    });
+
+    it("should capture the parse error", async () => {
+      emitter.on("collector:parse-error", ({ error }) => {
+        expect(error.type).toBe("Parse");
+        expect(error.message).toMatch(/end of the stream/);
+      });
+
+      const { collectFile } = createCollector(emitter, __dirname);
+      const file = await collectFile(
+        __dirname,
+        "./__tests__/sources/test/broken-frontmatter"
+      );
+      expect(file).toBeNull();
+    });
+  });
+
+  describe("collect", () => {
+    const { collect } = createCollector(emitter, __dirname);
+
+    it("should collect one collection", async () => {
+      const collections = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collections).toHaveLength(1);
+    });
+
+    it("should collect two files", async () => {
+      const [collection] = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collection?.files).toHaveLength(2);
+    });
+
+    it("should store file path", async () => {
+      const [collection] = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collection?.files[0]?.path).toMatch(/001\.md$/);
+      expect(collection?.files[1]?.path).toMatch(/002\.md$/);
+    });
+
+    it("should parse frontmatter", async () => {
+      const [collection] = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collection?.files[0]?.data.name).toBe("One");
+      expect(collection?.files[1]?.data.name).toBe("Two");
+    });
+
+    it("should store body", async () => {
+      const [collection] = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collection?.files[0]?.data.content.trim()).toBe("# One");
+      expect(collection?.files[1]?.data.content.trim()).toBe("# Two");
+    });
+
+    it("should collect single collection from multiple sources", async () => {
+      const [collection] = await collect([
+        {
+          directory: "./__tests__/sources/test",
+          include: ["001.md", "002.md"],
+        },
+      ]);
+
+      expect(collection?.files).toHaveLength(2);
+    });
+
+    it("should collect multiple collections", async () => {
+      const collections = await collect([
+        {
+          directory: "./__tests__/sources/test/",
+          include: "*.md",
+        },
+        {
+          directory: "./__tests__/sources/posts/",
+          include: "*.md",
+        },
+      ]);
+
+      expect(collections).toHaveLength(2);
+    });
+  });
+});

package/src/collector.ts

@@ -0,0 +1,114 @@
+import matter from "gray-matter";
+import fg from "fast-glob";
+import { readFile } from "fs/promises";
+import { AnyCollection } from "./config";
+import path from "path";
+import { isDefined } from "./utils";
+import { CollectionFile } from "./types";
+import { Emitter } from "./events";
+
+export type CollectorEvents = {
+  "collector:read-error": {
+    filePath: string;
+    error: CollectError;
+  };
+  "collector:parse-error": {
+    filePath: string;
+    error: CollectError;
+  };
+}
+
+export type ErrorType = "Parse" | "Read";
+
+export class CollectError extends Error {
+  type: ErrorType;
+  constructor(type: ErrorType, message: string) {
+    super(message);
+    this.type = type;
+  }
+}
+
+type FileCollection = Pick<AnyCollection, "directory" | "include">;
+
+export function createCollector(emitter: Emitter, baseDirectory: string = ".") {
+  async function read(filePath: string) {
+    try {
+      return await readFile(filePath, "utf-8");
+    } catch (error) {
+      emitter.emit("collector:read-error", {
+        filePath,
+        error: new CollectError("Read", String(error)),
+      });
+      return null;
+    }
+  }
+
+  function parse(file: string) {
+    const { data, content } = matter(file);
+
+    return {
+      ...data,
+      content
+    };
+  }
+
+  async function collectFile(
+    directory: string,
+    filePath: string
+  ): Promise<CollectionFile | null> {
+    const file = await read(path.join(directory, filePath));
+    if (!file) {
+      return null;
+    }
+
+    try {
+      const data = parse(file);
+
+      return {
+        data,
+        path: filePath,
+      };
+    } catch (error) {
+      emitter.emit("collector:parse-error", {
+        filePath: path.join(directory, filePath),
+        error: new CollectError("Parse", String(error)),
+      });
+      return null;
+    }
+  }
+
+  async function resolveCollection<T extends FileCollection>(collection: T) {
+    const collectionDirectory = path.join(baseDirectory, collection.directory);
+    const filePaths = await fg(collection.include, {
+      cwd: collectionDirectory,
+      onlyFiles: true,
+      absolute: false,
+    });
+    const promises = filePaths.map((filePath) =>
+      collectFile(collectionDirectory, filePath)
+    );
+
+    const files = await Promise.all(promises);
+
+    return {
+      ...collection,
+      files: files.filter(isDefined),
+    };
+  }
+
+  async function collect<T extends FileCollection>(
+    unresolvedCollections: Array<T>
+  ) {
+    const promises = unresolvedCollections.map((collection) =>
+      resolveCollection(collection)
+    );
+    return await Promise.all(promises);
+  }
+
+  return {
+    collect,
+    collectFile,
+  };
+}
+
+export type Collector = ReturnType<typeof createCollector>;

package/src/config.ts

@@ -0,0 +1,106 @@
+import { ZodObject, ZodRawShape, ZodString, ZodTypeAny, z } from "zod";
+import { generateTypeName } from "./utils";
+
+export type Meta = {
+  filePath: string;
+  fileName: string;
+  directory: string;
+  path: string;
+  extension: string;
+};
+
+type WithContent = {
+  content: ZodString;
+};
+
+type AddContent<TShape extends ZodRawShape> = TShape extends {
+  content: ZodTypeAny;
+}
+  ? TShape
+  : TShape & WithContent;
+
+export type Schema<TShape extends ZodRawShape> = z.infer<
+  ZodObject<AddContent<TShape>>
+> & {
+  _meta: Meta;
+};
+
+export type Context = {
+  documents<TCollection extends AnyCollection>(
+    collection: TCollection
+  ): Array<Schema<TCollection["schema"]>>;
+};
+
+type Z = typeof z;
+
+export type CollectionRequest<
+  TName extends string,
+  TShape extends ZodRawShape,
+  TSchema,
+  TTransformResult,
+  TDocument
+> = {
+  name: TName;
+  typeName?: string;
+  schema: (z: Z) => TShape;
+  transform?: (context: Context, data: TSchema) => TTransformResult;
+  directory: string;
+  include: string | string[];
+  onSuccess?: (documents: Array<TDocument>) => void | Promise<void>;
+};
+
+export type Collection<
+  TName extends string,
+  TShape extends ZodRawShape,
+  TSchema,
+  TTransformResult,
+  TDocument
+> = Omit<
+  CollectionRequest<TName, TShape, TSchema, TTransformResult, TDocument>,
+  "schema"
+> & {
+  typeName: string;
+  schema: TShape;
+};
+
+export type AnyCollection = Collection<any, ZodRawShape, any, any, any>;
+
+export function defineCollection<
+  TName extends string,
+  TShape extends ZodRawShape,
+  TSchema = Schema<TShape>,
+  TTransformResult = never,
+  TDocument = [TTransformResult] extends [never]
+    ? Schema<TShape>
+    : Awaited<TTransformResult>
+>(
+  collection: CollectionRequest<
+    TName,
+    TShape,
+    TSchema,
+    TTransformResult,
+    TDocument
+  >
+): Collection<TName, TShape, TSchema, TTransformResult, TDocument> {
+  let typeName = collection.typeName;
+  if (!typeName) {
+    typeName = generateTypeName(collection.name);
+  }
+  return {
+    ...collection,
+    typeName,
+    schema: collection.schema(z),
+  };
+}
+
+export type Configuration<TCollections extends Array<AnyCollection>> = {
+  collections: TCollections;
+};
+
+export type AnyConfiguration = Configuration<Array<AnyCollection>>;
+
+export function defineConfig<TConfig extends AnyConfiguration>(
+  config: TConfig
+) {
+  return config;
+}

package/src/configurationReader.test.ts

@@ -0,0 +1,104 @@
+import path from "path";
+import { describe, it, expect } from "vitest";
+import { createConfigurationReader } from "./configurationReader";
+import { existsSync } from "fs";
+import { z } from "zod";
+
+function config(name: string) {
+  const configPath = path.join(__dirname, "__tests__", name);
+  const readConfig = createConfigurationReader();
+  return readConfig(configPath, { configName: name + ".js" });
+}
+
+describe("configurationReader", () => {
+  it("should read the config", async () => {
+    const cfg = await config("config.001.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+  });
+
+  it("should have a typeName", async () => {
+    const cfg = await config("config.001.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+    expect(cfg.collections[0]?.typeName).toBe("Post");
+  });
+
+  it("should have a directory", async () => {
+    const cfg = await config("config.001.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+    expect(cfg.collections[0]?.directory).toBe("sources/posts");
+  });
+
+  it("should have a include pattern", async () => {
+    const cfg = await config("config.001.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+    expect(cfg.collections[0]?.include).toBe("**/*.md(x)?");
+  });
+
+  it("should generate a typeName", async () => {
+    const cfg = await config("config.002.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+    expect(cfg.collections[0]?.typeName).toBe("Post");
+  });
+
+  it("should read the config with an imported collection", async () => {
+    const cfg = await config("config.003.ts");
+
+    expect(cfg.collections).toHaveLength(1);
+    expect(cfg.collections[0]?.name).toBe("posts");
+  });
+
+  it("should have a valid zod schema", async () => {
+    const cfg = await config("config.002.ts");
+    const schema = cfg.collections[0]?.schema;
+    if (!schema) {
+      throw new Error("collection does not have a schema");
+    }
+
+    let result = z.object(schema).safeParse({
+      title: "Hello",
+    });
+    expect(result.success).toBe(true);
+
+    result = z.object(schema).safeParse({
+      greeting: "Hello",
+    });
+    expect(result.success).toBe(false);
+  });
+
+  it("should use different cache directory and name", async () => {
+    const basedir = path.join(__dirname, "__tests__");
+    const cacheDir = path.join(
+      basedir,
+      ".content-collections",
+      "different-cache-dir"
+    );
+    const configPath = path.join(basedir, "config.001.ts");
+    const configName = "different.config.js";
+    const readConfig = createConfigurationReader();
+    const cfg = await readConfig(configPath, {
+      configName,
+      cacheDir,
+    });
+
+    const compiledConfig = path.join(cacheDir, configName);
+    expect(cfg.collections).toHaveLength(1);
+    expect(existsSync(compiledConfig)).toBe(true);
+  });
+
+  it("should throw an error if the config file does not exists", async () => {
+    await expect(config("non-existing")).rejects.toThrowError(
+      /configuration file .*\/non-existing does not exist/
+    );
+  });
+
+  it("should throw an error if the config file is invalid", async () => {
+    await expect(config("invalid")).rejects.toThrowError(
+      /configuration file .*\/invalid is invalid/
+    );
+  });
+});

package/src/configurationReader.ts

@@ -0,0 +1,108 @@
+import * as esbuild from "esbuild";
+import fs from "node:fs/promises";
+import path from "node:path";
+import packageJson from "../package.json";
+import { AnyCollection } from "./config";
+import { existsSync } from "node:fs";
+
+export type ErrorType = "Read" | "Compile";
+
+export class ConfigurationError extends Error {
+  type: ErrorType;
+  constructor(type: ErrorType, message: string) {
+    super(message);
+    this.type = type;
+  }
+}
+
+export type InternalConfiguration = {
+  collections: Array<AnyCollection>;
+  path: string;
+  generateTypes?: boolean;
+};
+
+const importPathPlugin: esbuild.Plugin = {
+  name: "import-path",
+  setup(build) {
+    build.onResolve({ filter: /^\@content-collections\/core$/ }, () => {
+      return { path: path.join(__dirname, "index.ts"), external: true };
+    });
+  },
+};
+
+export type Options = {
+  configName: string;
+  cacheDir?: string;
+};
+
+export const defaultConfigName = "content-collection-config.mjs";
+
+function resolveCacheDir(config: string, options: Options) {
+  if (options.cacheDir) {
+    return options.cacheDir;
+  }
+  return path.join(path.dirname(config), ".content-collections", "cache");
+}
+
+async function compile(configurationPath: string, outfile: string) {
+  const plugins: Array<esbuild.Plugin> = [];
+  if (process.env.NODE_ENV === "test") {
+    plugins.push(importPathPlugin);
+  }
+
+  await esbuild.build({
+    entryPoints: [configurationPath],
+    external: [...Object.keys(packageJson.dependencies), "@content-collections/*"],
+    bundle: true,
+    platform: "node",
+    format: "esm",
+    plugins,
+    outfile,
+  });
+}
+
+// errorHandler does not make sense here:
+// because if the configuration is invalid, the program should exit
+
+export function createConfigurationReader() {
+  return async (
+    configurationPath: string,
+    options: Options = {
+      configName: defaultConfigName,
+    }
+  ): Promise<InternalConfiguration> => {
+    if (!existsSync(configurationPath)) {
+      throw new ConfigurationError(
+        "Read",
+        `configuration file ${configurationPath} does not exist`
+      );
+    }
+
+    const cacheDir = resolveCacheDir(configurationPath, options);
+    await fs.mkdir(cacheDir, { recursive: true });
+
+    const outfile = path.join(cacheDir, options.configName);
+
+    try {
+      await compile(configurationPath, outfile);
+    } catch (error) {
+      throw new ConfigurationError(
+        "Compile",
+        `configuration file ${configurationPath} is invalid: ${error}`
+      );
+    }
+
+    const module = await import(
+      `file://${path.resolve(outfile)}?x=${Date.now()}`
+    );
+    return {
+      ...module.default,
+      path: configurationPath,
+      generateTypes: true,
+    };
+  };
+}
+
+export type ConfigurationReader = Awaited<
+  ReturnType<typeof createConfigurationReader>
+>;

package/src/events.test.ts

@@ -0,0 +1,84 @@
+import { describe, it, expect } from "vitest";
+import { createEmitter } from "./events";
+
+type Events = {
+  test: {
+    a: string;
+  };
+  other: {
+    error: Error
+  }
+};
+
+describe("events", () => {
+  it("should emit and listen to events", () => {
+    const emitter = createEmitter<Events>();
+    emitter.on("test", (event) => {
+      expect(event.a).toBe("test");
+    });
+    emitter.emit("test", { a: "test" });
+  });
+
+  it("should only allow keys from the event map", () => {
+    const emitter = createEmitter<Events>();
+    // @ts-expect-error test 2 does not exist
+    emitter.emit("test2", { a: "test" });
+  });
+
+  it("should only allow listeners for keys from the event map", () => {
+    const emitter = createEmitter<Events>();
+    // @ts-expect-error test 2 does not exist
+    emitter.on("test2", (event) => {});
+  });
+
+  it("should only correct event type on emit", () => {
+    const emitter = createEmitter<Events>();
+    // @ts-expect-error b does not exist
+    emitter.emit("test", { b: "test" });
+  });
+
+  it("should only allow listeners with the correct event type", () => {
+    const emitter = createEmitter<Events>();
+    emitter.on("test", (event) => {
+      // @ts-expect-error b does not exist
+      expect(event.b).toBeUndefined();
+    });
+  });
+
+  it("should emit extra collection-error on error event", () => {
+    const emitter = createEmitter<Events>();
+
+    const events: Array<string> = [];
+
+    emitter.on("other", (event) => {
+      events.push("other");
+    });
+    emitter.on("_error", (event) => {
+      events.push("error");
+      expect(event._event).toBe("other");
+      expect(event.error.message).toBe("test");
+    });
+
+    emitter.emit("other", { error: new Error("test") });
+
+    expect(events).toEqual(["other", "error"]);
+  });
+
+  it("should emit extra _all event", () => {
+    const emitter = createEmitter<Events>();
+
+    const events: Array<string> = [];
+
+    emitter.on("test", (event) => {
+      events.push("test");
+    });
+    emitter.on("_all", (event) => {
+      events.push("all");
+      expect(event._event).toBe("test");
+    });
+
+    emitter.emit("test", { a: "test" });
+
+    expect(events).toEqual(["test", "all"]);
+  });
+});