@content-collections/core 0.1.0

package/src/__tests__/.content-collections/cache/config.002.ts.js

@@ -0,0 +1,16 @@
+// src/__tests__/config.002.ts
+import { defineCollection, defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+var posts = defineCollection({
+  name: "posts",
+  schema: (z) => ({
+    title: z.string()
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?"
+});
+var config_002_default = defineConfig({
+  collections: [posts]
+});
+export {
+  config_002_default as default
+};

package/src/__tests__/.content-collections/cache/config.003.ts.js

@@ -0,0 +1,24 @@
+// src/__tests__/config.003.ts
+import { defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+
+// src/__tests__/collections/posts.ts
+import { defineCollection } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+var posts_default = defineCollection({
+  name: "posts",
+  typeName: "Post",
+  schema: (z) => ({
+    title: z.string().min(5),
+    description: z.string().min(10),
+    date: z.union([z.string().regex(/^\d{4}-\d{2}-\d{2}$/), z.date()]).transform((val) => new Date(val))
+  }),
+  directory: "posts",
+  include: "**/*.md(x)?"
+});
+
+// src/__tests__/config.003.ts
+var config_003_default = defineConfig({
+  collections: [posts_default]
+});
+export {
+  config_003_default as default
+};

package/src/__tests__/.content-collections/cache/config.004.mjs

@@ -0,0 +1,47 @@
+// src/__tests__/config.004.ts
+import { defineCollection, defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+import fs from "node:fs";
+function mkdir(directory) {
+  if (fs.existsSync(directory)) {
+    return;
+  }
+  fs.mkdirSync(directory);
+}
+var posts = defineCollection({
+  name: "posts",
+  schema: (z) => ({
+    title: z.string()
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?",
+  onSuccess: (documents) => {
+    mkdir("tmp");
+    fs.writeFileSync(
+      "tmp/posts.length",
+      JSON.stringify(documents.length),
+      "utf-8"
+    );
+  }
+});
+var authors = defineCollection({
+  name: "authors",
+  schema: (z) => ({
+    displayName: z.string()
+  }),
+  directory: "sources/authors",
+  include: "**/*.md(x)?",
+  onSuccess: (documents) => {
+    mkdir("tmp");
+    fs.writeFileSync(
+      "tmp/authors.length",
+      JSON.stringify(documents.length),
+      "utf-8"
+    );
+  }
+});
+var config_004_default = defineConfig({
+  collections: [posts, authors]
+});
+export {
+  config_004_default as default
+};

package/src/__tests__/.content-collections/different-cache-dir/different.config.js

@@ -0,0 +1,19 @@
+// src/__tests__/config.001.ts
+import { defineCollection, defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+var posts = defineCollection({
+  name: "posts",
+  typeName: "Post",
+  schema: (z) => ({
+    title: z.string().min(5),
+    description: z.string().min(10),
+    date: z.union([z.string().regex(/^\d{4}-\d{2}-\d{2}$/), z.date()]).transform((val) => new Date(val))
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?"
+});
+var config_001_default = defineConfig({
+  collections: [posts]
+});
+export {
+  config_001_default as default
+};

package/src/__tests__/.content-collections/generated-config.002/allPosts.json

@@ -0,0 +1,13 @@
+[
+  {
+    "content": "\n# First post",
+    "title": "First post",
+    "_meta": {
+      "filePath": "first.md",
+      "fileName": "first.md",
+      "directory": ".",
+      "extension": "md",
+      "path": "first"
+    }
+  }
+]

package/src/__tests__/.content-collections/generated-config.002/index.d.ts

@@ -0,0 +1,7 @@
+import configuration from "../../config.002.ts";
+import { GetTypeByName } from "@content-collections/core";
+
+export type Post = GetTypeByName<typeof configuration, "posts">;
+export declare const allPosts: Array<Post>;
+
+export {};

package/src/__tests__/.content-collections/generated-config.002/index.js

@@ -0,0 +1,5 @@
+// generated by content-collections at Mon Jan 15 2024 19:52:01 GMT+0100 (GMT+01:00)
+
+import allPosts from "./allPosts.json";
+
+export { allPosts };

package/src/__tests__/.content-collections/generated-config.004/allAuthors.json

@@ -0,0 +1,13 @@
+[
+  {
+    "content": "\nTricia Marie McMillan, also known as Trillian Astra, is a fictional character from Douglas Adams' series The Hitchhiker's Guide to the Galaxy. She is most commonly referred to simply as \"Trillian\", a modification of her birth name, which she adopted because it sounded more \"space-like\". According to the movie version, her middle name is Marie. Physically, she is described as \"a slim, darkish humanoid, with long waves of black hair, a full mouth, an odd little knob of a nose and ridiculously brown eyes,\" looking \"vaguely Arabic.\"\n\nSource: https://en.wikipedia.org/wiki/Trillian_(character)",
+    "displayName": "Tricia Marie McMillan",
+    "_meta": {
+      "filePath": "trillian.md",
+      "fileName": "trillian.md",
+      "directory": ".",
+      "extension": "md",
+      "path": "trillian"
+    }
+  }
+]

package/src/__tests__/.content-collections/generated-config.004/allPosts.json

@@ -0,0 +1,13 @@
+[
+  {
+    "content": "\n# First post",
+    "title": "First post",
+    "_meta": {
+      "filePath": "first.md",
+      "fileName": "first.md",
+      "directory": ".",
+      "extension": "md",
+      "path": "first"
+    }
+  }
+]

package/src/__tests__/.content-collections/generated-config.004/index.d.ts

@@ -0,0 +1,10 @@
+import configuration from "../../config.004.ts";
+import { GetTypeByName } from "@content-collections/core";
+
+export type Post = GetTypeByName<typeof configuration, "posts">;
+export declare const allPosts: Array<Post>;
+
+export type Author = GetTypeByName<typeof configuration, "authors">;
+export declare const allAuthors: Array<Author>;
+
+export {};

package/src/__tests__/.content-collections/generated-config.004/index.js

@@ -0,0 +1,6 @@
+// generated by content-collections at Mon Jan 15 2024 19:52:01 GMT+0100 (GMT+01:00)
+
+import allPosts from "./allPosts.json";
+import allAuthors from "./allAuthors.json";
+
+export { allPosts, allAuthors };

package/src/__tests__/collections/posts.ts

@@ -0,0 +1,15 @@
+import { defineCollection } from "@content-collections/core";
+
+export default defineCollection({
+  name: "posts",
+  typeName: "Post",
+  schema: (z) => ({
+    title: z.string().min(5),
+    description: z.string().min(10),
+    date: z
+      .union([z.string().regex(/^\d{4}-\d{2}-\d{2}$/), z.date()])
+      .transform((val) => new Date(val)),
+  }),
+  directory: "posts",
+  include: "**/*.md(x)?",
+});

package/src/__tests__/config.001.ts

@@ -0,0 +1,19 @@
+import { defineCollection, defineConfig } from "@content-collections/core";
+
+const posts = defineCollection({
+  name: "posts",
+  typeName: "Post",
+  schema: (z) => ({
+    title: z.string().min(5),
+    description: z.string().min(10),
+    date: z
+      .union([z.string().regex(/^\d{4}-\d{2}-\d{2}$/), z.date()])
+      .transform((val) => new Date(val)),
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?",
+});
+
+export default defineConfig({
+  collections: [posts],
+});

package/src/__tests__/config.002.ts

@@ -0,0 +1,14 @@
+import { defineCollection, defineConfig } from "@content-collections/core";
+
+const posts = defineCollection({
+  name: "posts",
+  schema: (z) => ({
+    title: z.string(),
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?",
+});
+
+export default defineConfig({
+  collections: [posts],
+});

package/src/__tests__/config.003.ts

@@ -0,0 +1,6 @@
+import { defineConfig } from "@content-collections/core";
+import posts from "./collections/posts";
+
+export default defineConfig({
+  collections: [posts],
+});

package/src/__tests__/config.004.ts

@@ -0,0 +1,47 @@
+import { defineCollection, defineConfig } from "@content-collections/core";
+import fs from "node:fs";
+
+function mkdir(directory: string) {
+  if (fs.existsSync(directory)) {
+    return;
+  }
+  fs.mkdirSync(directory);
+}
+
+const posts = defineCollection({
+  name: "posts",
+  schema: (z) => ({
+    title: z.string(),
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?",
+  onSuccess: (documents) => {
+    mkdir("tmp");
+    fs.writeFileSync(
+      "tmp/posts.length",
+      JSON.stringify(documents.length),
+      "utf-8"
+    );
+  },
+});
+
+const authors = defineCollection({
+  name: "authors",
+  schema: (z) => ({
+    displayName: z.string(),
+  }),
+  directory: "sources/authors",
+  include: "**/*.md(x)?",
+  onSuccess: (documents) => {
+    mkdir("tmp");
+    fs.writeFileSync(
+      "tmp/authors.length",
+      JSON.stringify(documents.length),
+      "utf-8"
+    );
+  },
+});
+
+export default defineConfig({
+  collections: [posts, authors],
+});

package/src/__tests__/invalid

@@ -0,0 +1 @@
+This is not valid TypeScript code.

package/src/__tests__/sources/authors/trillian.md

@@ -0,0 +1,8 @@
+---
+ref: trillian
+displayName: Tricia Marie McMillan
+---
+
+Tricia Marie McMillan, also known as Trillian Astra, is a fictional character from Douglas Adams' series The Hitchhiker's Guide to the Galaxy. She is most commonly referred to simply as "Trillian", a modification of her birth name, which she adopted because it sounded more "space-like". According to the movie version, her middle name is Marie. Physically, she is described as "a slim, darkish humanoid, with long waves of black hair, a full mouth, an odd little knob of a nose and ridiculously brown eyes," looking "vaguely Arabic."
+
+Source: https://en.wikipedia.org/wiki/Trillian_(character)

package/src/__tests__/sources/posts/first.md

@@ -0,0 +1,6 @@
+---
+title: First post
+author: trillian
+---
+
+# First post

package/src/__tests__/sources/test/001.md

@@ -0,0 +1,5 @@
+---
+name: One
+---
+
+# One

package/src/__tests__/sources/test/002.md

@@ -0,0 +1,5 @@
+---
+name: Two
+---
+
+# Two

package/src/__tests__/sources/test/broken-frontmatter

@@ -0,0 +1,6 @@
+---
+t"
+author: trillian
+---
+
+# One

package/src/builder.test.ts

@@ -0,0 +1,180 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { createBuilder as origCreateBuilder } from "./builder";
+import path from "node:path";
+import { existsSync } from "node:fs";
+import fs from "node:fs/promises";
+import { Emitter } from "./events";
+
+// we mock the watcher module, because it causes problems in some situations
+// we test the watcher module separately
+vi.mock("./watcher", async () => {
+  return {
+    createWatcher: async (_: Emitter, paths: Array<string>) => {
+      return {
+        paths,
+        unsubscribe: async () => {},
+      };
+    },
+  };
+});
+
+describe("builder", () => {
+  afterEach(async () => {
+    if (existsSync("tmp")) {
+      await fs.rm("tmp", { recursive: true });
+    }
+  });
+
+  async function createBuilder(name: string) {
+    const configPath = path.join(__dirname, "__tests__", name + ".ts");
+    const outputDir = path.join(
+      __dirname,
+      "__tests__",
+      ".content-collections",
+      "generated-" + name
+    );
+    const builder = await origCreateBuilder(configPath, {
+      configName: name + ".mjs",
+      outputDir,
+    });
+    return {
+      builder,
+      outputDir,
+    };
+  }
+
+  describe("build", () => {
+    it("should build", async () => {
+      const { builder, outputDir } = await createBuilder("config.002");
+      await builder.build();
+
+      const { allPosts } = await import(path.resolve(outputDir, "index.js"));
+      expect(allPosts.length).toBe(1);
+    });
+
+    it("should call onSuccess", async () => {
+      const { builder, outputDir } = await createBuilder("config.004");
+      await builder.build();
+
+      const { allPosts, allAuthors } = await import(
+        path.resolve(outputDir, "index.js")
+      );
+
+      expect(allPosts.length).toBe(1);
+      const postsLength = await fs.readFile(
+        path.join("tmp", "posts.length"),
+        "utf-8"
+      );
+      expect(postsLength).toBe("1");
+
+      expect(allAuthors.length).toBe(1);
+      const authorsLength = await fs.readFile(
+        path.join("tmp", "authors.length"),
+        "utf-8"
+      );
+      expect(authorsLength).toBe("1");
+    });
+
+    it("should emit build:start and build:end", async () => {
+      const { builder } = await createBuilder("config.002");
+      const events: Array<string> = [];
+
+      builder.on("builder:start", (event) => {
+        events.push("builder:start");
+        expect(event.startedAt).toBeDefined();
+      });
+      builder.on("builder:end", (event) => {
+        events.push("builder:end");
+        expect(event.startedAt).toBeDefined();
+        expect(event.endedAt).toBeDefined();
+      });
+
+      await builder.build();
+
+      expect(events).toEqual(["builder:start", "builder:end"]);
+    });
+  });
+
+  describe("sync", () => {
+    beforeEach(async () => {
+      const sourceDir = path.join("tmp", "sources", "posts");
+      if (!existsSync(sourceDir)) {
+        await fs.mkdir(sourceDir, { recursive: true });
+      }
+
+      const configPath = path.join(__dirname, "__tests__", "config.002.ts");
+      await fs.copyFile(configPath, path.join("tmp", "config.ts"));
+      await fs.copyFile(
+        path.join(__dirname, "__tests__", "sources", "posts", "first.md"),
+        path.join(sourceDir, "first.md")
+      );
+    });
+
+    it("should sync new files", async () => {
+      const builder = await origCreateBuilder(path.join("tmp", "config.ts"), {
+        configName: "sync.new.file.ts",
+        cacheDir: path.join("tmp", "cache"),
+        outputDir: path.join("tmp", "output-new"),
+      });
+
+      await builder.build();
+
+      const newFile = path.join("tmp", "sources", "posts", "second.md");
+      await fs.writeFile(newFile, "---\ntitle: Second\n---\nSecond post");
+
+      const synced = await builder.sync("create", newFile);
+      expect(synced).toBe(true);
+
+      await builder.build();
+
+      const { allPosts } = await import(
+        path.resolve("tmp", "output-new", "index.js")
+      );
+
+      expect(allPosts.length).toBe(2);
+    });
+
+    it("should sync deleted files", async () => {
+      const builder = await origCreateBuilder(path.join("tmp", "config.ts"), {
+        configName: "sync.rm.file.ts",
+        cacheDir: path.join("tmp", "cache"),
+        outputDir: path.join("tmp", "output-rm"),
+      });
+
+      await builder.build();
+
+      const rmFile = path.join("tmp", "sources", "posts", "first.md");
+      await fs.rm(rmFile);
+
+      const synced = await builder.sync("delete", rmFile);
+      expect(synced).toBe(true);
+
+      await builder.build();
+
+      const { allPosts } = await import(
+        path.resolve("tmp", "output-rm", "index.js")
+      );
+
+      expect(allPosts.length).toBe(0);
+    });
+  });
+
+  describe("watch", () => {
+    it("should return watcher", async () => {
+      const { builder } = await createBuilder("config.002");
+
+      const watcher = await builder.watch();
+      expect(watcher).toBeTruthy();
+    });
+
+    it("should pass the collection directories to the watcher", async () => {
+      const { builder } = await createBuilder("config.002");
+
+      const watcher = await builder.watch();
+      // @ts-ignore our mock returns a watcher with a paths property
+      expect(watcher.paths).toEqual([
+        path.join(__dirname, "__tests__", "sources", "posts"),
+      ]);
+    })
+  });
+});

package/src/builder.ts

@@ -0,0 +1,113 @@
+import {
+  createConfigurationReader,
+  Options as ConfigurationOptions,
+  defaultConfigName,
+} from "./configurationReader";
+import { createCollector } from "./collector";
+import { Modification } from "./types";
+import { createWriter } from "./writer";
+import { createTransformer } from "./transformer";
+import { isDefined } from "./utils";
+import { createSynchronizer } from "./synchronizer";
+import path from "node:path";
+import { createWatcher } from "./watcher";
+import { Events, createEmitter } from "./events";
+
+export type BuilderEvents = {
+  "builder:start": {
+    startedAt: number;
+  };
+  "builder:end": {
+    startedAt: number;
+    endedAt: number;
+  };
+};
+
+type Options = ConfigurationOptions & {
+  outputDir?: string;
+};
+
+function resolveOutputDir(baseDirectory: string, options: Options) {
+  if (options.outputDir) {
+    return options.outputDir;
+  }
+  return path.join(baseDirectory, ".content-collections", "generated");
+}
+
+export async function createBuilder(
+  configurationPath: string,
+  options: Options = {
+    configName: defaultConfigName,
+  }
+) {
+  const emitter = createEmitter<Events>();
+
+  const readConfiguration = createConfigurationReader();
+  const configuration = await readConfiguration(configurationPath, options);
+  const baseDirectory = path.dirname(configurationPath);
+  const directory = resolveOutputDir(baseDirectory, options);
+
+  // TODO: we should not collect files before the user has a chance to register listeners
+  const collector = createCollector(emitter, baseDirectory);
+  const writer = await createWriter(directory);
+
+  const [resolved] = await Promise.all([
+    collector.collect(configuration.collections),
+    writer.createJavaScriptFile(configuration),
+    writer.createTypeDefinitionFile(configuration),
+  ]);
+
+  const synchronizer = createSynchronizer(
+    collector.collectFile,
+    resolved,
+    baseDirectory
+  );
+  const transform = createTransformer(emitter);
+
+  async function sync(modification: Modification, filePath: string) {
+    if (modification === "delete") {
+      return synchronizer.deleted(filePath);
+    }
+    return synchronizer.changed(filePath);
+  }
+
+  async function build() {
+    const startedAt = Date.now();
+    emitter.emit("builder:start", {
+      startedAt,
+    });
+
+    const collections = await transform(resolved);
+    await writer.createDataFiles(collections);
+
+    const pendingOnSuccess = collections
+      .filter((collection) => Boolean(collection.onSuccess))
+      .map((collection) =>
+        collection.onSuccess?.(collection.documents.map((doc) => doc.document))
+      );
+
+    await Promise.all(pendingOnSuccess.filter(isDefined));
+
+    emitter.emit("builder:end", {
+      startedAt,
+      endedAt: Date.now(),
+    });
+  }
+
+  async function watch() {
+    const paths = resolved.map((collection) =>
+      path.join(baseDirectory, collection.directory)
+    );
+    const watcher = await createWatcher(emitter, paths, sync, build);
+    return watcher;
+  }
+
+  return {
+    sync,
+    build,
+    watch,
+    on: emitter.on,
+  };
+}
+
+export type Builder = Awaited<ReturnType<typeof createBuilder>>;