@content-collections/core 0.1.0

package/src/types.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, it } from "vitest";
+import { GetTypeByName } from "./types";
+import { defineCollection, defineConfig } from "./config";
+
+describe("types", () => {
+  describe("GetTypeByName", () => {
+    it("should infer type from schema", () => {
+      const collection = defineCollection({
+        name: "person",
+        typeName: "person",
+        directory: "./persons",
+        include: "*.md",
+        schema: (z) => ({
+          name: z.string(),
+          age: z.number(),
+        }),
+      });
+
+      const config = defineConfig({
+        collections: [collection],
+      });
+
+      type Person = GetTypeByName<typeof config, "person">;
+
+      const person: Person = {
+        name: "John",
+        age: 20,
+        // @ts-expect-error city is not in the schema
+        city: "New York",
+        _meta: {
+          fileName: "john.md",
+          filePath: "persons/john.md",
+          directory: "persons",
+          path: "persons/john",
+          extension: "md",
+        },
+      };
+
+      expect(person).toBeTruthy();
+    });
+
+    it("should infer type from transform function", () => {
+      const collection = defineCollection({
+        name: "person",
+        typeName: "person",
+        directory: "./persons",
+        include: "*.md",
+        schema: (z) => ({
+          name: z.string(),
+          age: z.number(),
+        }),
+        transform: (ctx, data) => {
+          return {
+            ...data,
+            city: "New York",
+          };
+        },
+      });
+
+      const config = defineConfig({
+        collections: [collection],
+      });
+
+      type Person = GetTypeByName<typeof config, "person">;
+      const person: Person = {
+        name: "John",
+        city: "New York",
+        // @ts-expect-error street is not in the schema
+        street: "Main Street",
+        _meta: {
+          fileName: "john.md",
+          filePath: "persons/john.md",
+          directory: "persons",
+          path: "persons/john",
+          extension: "md",
+        },
+      };
+
+      expect(person).toBeTruthy();
+    });
+  });
+
+  it("should infer type from other collection", () => {
+    const countryCollection = defineCollection({
+      name: "country",
+      directory: "./countries",
+      include: "*.md",
+      schema: (z) => ({
+        code: z.string().length(2),
+        name: z.string(),
+      }),
+    });
+
+    const personCollection = defineCollection({
+      name: "person",
+      directory: "./persons",
+      include: "*.md",
+      schema: (z) => ({
+        name: z.string(),
+        age: z.number(),
+        countryCode: z.string().length(2),
+      }),
+      transform: (ctx, data) => {
+        const countries = ctx.documents(countryCollection);
+        const country = countries.find((c) => c.code === data.countryCode);
+        if (!country) {
+          throw new Error(`Country ${data.countryCode} not found`);
+        }
+        return {
+          name: data.name,
+          age: data.age,
+          country: {
+            code: country.code,
+            name: country.name,
+          }
+        };
+      },
+    });
+
+    const config = defineConfig({
+      collections: [countryCollection, personCollection],
+    });
+
+    type Person = GetTypeByName<typeof config, "person">;
+
+    const person: Person = {
+      name: "Hans",
+      age: 20,
+      country: {
+        code: "de",
+        name: "Germany",
+      },
+    };
+
+    expect(person).toBeTruthy();
+  });
+});

package/src/types.ts

@@ -0,0 +1,33 @@
+import { ZodRawShape } from "zod";
+import { AnyCollection, AnyConfiguration, Collection } from "./config";
+
+export type Modification = "create" | "update" | "delete";
+
+export type CollectionFile = {
+  data: {
+    content: string;
+    [key: string]: unknown;
+  };
+  path: string;
+};
+
+export type FileCollection = Pick<AnyCollection, "directory" | "include">;
+
+export type ResolvedCollection<T extends FileCollection> = T & {
+  files: Array<CollectionFile>;
+};
+
+type CollectionByName<TConfiguration extends AnyConfiguration> = {
+  [TCollection in TConfiguration["collections"][number] as TCollection["name"]]: TCollection;
+};
+
+type GetDocument<TCollection extends AnyCollection> =
+  TCollection extends Collection<any, ZodRawShape, any, any, infer TDocument>
+    ? TDocument
+    : never;
+
+export type GetTypeByName<
+  TConfiguration extends AnyConfiguration,
+  TName extends keyof CollectionByName<TConfiguration>,
+  TCollection = CollectionByName<TConfiguration>[TName]
+> = TCollection extends AnyCollection ? GetDocument<TCollection> : never;

package/src/utils.test.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect } from "vitest";
+import { generateTypeName, isDefined } from "./utils";
+
+describe("generateTypeName", () => {
+
+  it("should return same as collection name", () => {
+    const typeName = generateTypeName("Post");
+    expect(typeName).toBe("Post");
+  });
+
+  it("should return typeName with first letter upper case", () => {
+    const typeName = generateTypeName("post");
+    expect(typeName).toBe("Post");
+  });
+
+  it("should remove spaces", () => {
+    const typeName = generateTypeName("post title");
+    expect(typeName).toBe("PostTitle");
+  });
+
+  it("should return the singular form", () => {
+    const typeName = generateTypeName("posts");
+    expect(typeName).toBe("Post");
+  });
+
+});
+
+describe("isDefined", () => {
+
+  it("should filter null values", () => {
+    const values = [1, 2, null, 3, null];
+    const defined = values.filter(isDefined);
+    expect(defined).toEqual([1, 2, 3]);
+  });
+
+  it("should filter undefined values", () => {
+    const values = [1, 2, undefined, 3, undefined];
+    const defined = values.filter(isDefined);
+    expect(defined).toEqual([1, 2, 3]);
+  });
+
+  it("should filter null and undefined values", () => {
+    const values = [1, 2, undefined, 3, null];
+    const defined = values.filter(isDefined);
+    expect(defined).toEqual([1, 2, 3]);
+  });
+
+});

package/src/utils.ts

@@ -0,0 +1,11 @@
+import camelcase from "camelcase";
+import pluralize from "pluralize";
+
+export function generateTypeName(name: string) {
+  const singularName = pluralize.singular(name);
+  return camelcase(singularName, { pascalCase: true });
+}
+
+export function isDefined<T>(value: T | undefined | null): value is T {
+  return value !== undefined && value !== null;
+}

package/src/watcher.test.ts

@@ -0,0 +1,200 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { createWatcher } from "./watcher";
+import { Modification } from "./types";
+import { Events, createEmitter } from "./events";
+
+const isEnabled = process.env.ENABLE_WATCHER_TESTS === "true";
+
+describe.runIf(isEnabled)("watcher", () => {
+  const directories: Array<string> = [];
+
+  const events: Array<string> = [];
+  let build = false;
+
+  async function syncFn(
+    modification: Modification,
+    path: string
+  ): Promise<boolean> {
+    events.push(`${modification}:${path}`);
+    return !path.endsWith("-ignore");
+  }
+
+  async function buildFn() {
+    build = true;
+    return;
+  }
+
+  function findEvent(type: string, path: string, eventArray = events) {
+    return eventArray.find(
+      (event) => event.startsWith(`${type}:`) && event.endsWith(path)
+    );
+  }
+
+  let emitter = createEmitter<Events>();
+
+  beforeEach(() => {
+    emitter = createEmitter<Events>();
+    events.length = 0;
+    build = false;
+  });
+
+  afterEach(async () => {
+    for (const directory of directories.reverse()) {
+      if (existsSync(directory)) {
+        await fs.rm(directory, { recursive: true });
+      }
+    }
+    directories.length = 0;
+  });
+
+  const mkdir = async (path: string) => {
+    if (existsSync(path)) {
+      await fs.rm(path, { recursive: true });
+    }
+    await fs.mkdir(path);
+
+    directories.push(path);
+  };
+
+  it("should not build", async () => {
+    await mkdir("tmp");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+
+    await watcher.unsubscribe();
+
+    expect(build).toBe(false);
+  });
+
+  it("should not build for untracked file", async () => {
+    await mkdir("tmp");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+
+    // we create one file and wait for the event
+    // because we receive often a event for the directory
+    await fs.writeFile("tmp/foo-ignore", "foo");
+    await vi.waitUntil(() => findEvent("create", "tmp/foo-ignore"));
+
+    // we reset the build flag and create a second file
+    build = false;
+    await fs.writeFile("tmp/bar-ignore", "foo");
+    await vi.waitUntil(() => findEvent("create", "tmp/bar-ignore"));
+
+    await watcher.unsubscribe();
+
+    expect(build).toBe(false);
+  });
+
+  it("should emit create event", async () => {
+    await mkdir("tmp");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+    await fs.writeFile("tmp/foo", "foo");
+
+    await vi.waitUntil(() => findEvent("create", "tmp/foo"));
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("create", "tmp/foo")).toBeTruthy();
+    expect(build).toBe(true);
+  });
+
+  it("should emit update event", async () => {
+    await mkdir("tmp");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+
+    await fs.writeFile("tmp/foo", "foo", "utf-8");
+    await vi.waitUntil(() => findEvent("create", "tmp/foo"), 2000);
+
+    await fs.writeFile("tmp/foo", "bar", "utf-8");
+    await vi.waitUntil(() => findEvent("update", "tmp/foo"), 2000);
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("update", "tmp/foo")).toBeTruthy();
+    expect(build).toBe(true);
+  });
+
+  it("should emit delete event", async () => {
+    await mkdir("tmp");
+    await fs.writeFile("tmp/foo", "foo");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+    await fs.rm("tmp/foo");
+
+    await vi.waitUntil(() => findEvent("delete", "tmp/foo"));
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("delete", "tmp/foo")).toBeTruthy();
+    expect(build).toBe(true);
+  });
+
+  it("should emit events from multiple directories", async () => {
+    await mkdir("tmp-one");
+    await mkdir("tmp-two");
+
+    const watcher = await createWatcher(
+      emitter,
+      ["tmp-one", "tmp-two"],
+      syncFn,
+      buildFn
+    );
+    await fs.writeFile("tmp-one/foo", "foo");
+    await fs.writeFile("tmp-two/bar", "bar");
+
+    await vi.waitUntil(
+      () =>
+        findEvent("create", "tmp-one/foo") && findEvent("create", "tmp-two/bar")
+    );
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("create", "tmp-one/foo")).toBeTruthy();
+    expect(findEvent("create", "tmp-two/bar")).toBeTruthy();
+    expect(build).toBe(true);
+  });
+
+  it("should emit events from nested directories", async () => {
+    await mkdir("tmp");
+    await mkdir("tmp/foo");
+    await mkdir("tmp/bar");
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+    await fs.writeFile("tmp/foo/baz", "baz");
+    await fs.writeFile("tmp/bar/qux", "qux");
+
+    await vi.waitUntil(
+      () =>
+        findEvent("create", "tmp/foo/baz") && findEvent("create", "tmp/bar/qux")
+    );
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("create", "tmp/foo/baz")).toBeTruthy();
+    expect(findEvent("create", "tmp/bar/qux")).toBeTruthy();
+    expect(build).toBe(true);
+  });
+
+  it("should file change event to event emitter", async () => {
+    await mkdir("tmp");
+
+    const localEvents: Array<string> = [];
+    emitter.on("watcher:file-changed", ({ modification, filePath }) =>
+      localEvents.push(`${modification}:${filePath}`)
+    );
+
+    const watcher = await createWatcher(emitter, ["tmp"], syncFn, buildFn);
+    await fs.writeFile("tmp/foo", "foo");
+
+    await vi.waitUntil(() => findEvent("create", "tmp/foo", localEvents));
+
+    await watcher.unsubscribe();
+
+    expect(findEvent("create", "tmp/foo", localEvents)).toBeTruthy();
+  });
+});

package/src/watcher.ts

@@ -0,0 +1,56 @@
+import watcher, { SubscribeCallback } from "@parcel/watcher";
+import { Modification } from "./types";
+import { Emitter } from "./events";
+
+export type WatcherEvents = {
+  "watcher:file-changed": {
+    filePath: string;
+    modification: Modification;
+  };
+};
+
+type SyncFn = (modification: Modification, path: string) => Promise<boolean>;
+type BuildFn = () => Promise<void>;
+
+export async function createWatcher(
+  emitter: Emitter,
+  paths: Array<string>,
+  sync: SyncFn,
+  build: BuildFn
+) {
+  const onChange: SubscribeCallback = async (error, events) => {
+    if (error) {
+      console.error(error);
+      return;
+    }
+
+    let rebuild = false;
+
+    for (const event of events) {
+      if (await sync(event.type, event.path)) {
+        emitter.emit("watcher:file-changed", {
+          filePath: event.path,
+          modification: event.type,
+        });
+        rebuild = true;
+      }
+    }
+
+    if (rebuild) {
+      await build();
+    }
+  };
+
+  const subscriptions = await Promise.all(
+    paths.map((path) => watcher.subscribe(path, onChange))
+  );
+
+  return {
+    unsubscribe: async () => {
+      await Promise.all(
+        subscriptions.map((subscription) => subscription.unsubscribe())
+      );
+      return;
+    },
+  };
+}

package/src/writer.test.ts

@@ -0,0 +1,135 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { Writer, createWriter } from "./writer";
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+
+describe("writer", () => {
+  let writer: Writer;
+
+  beforeEach(async () => {
+    if (existsSync("tmp")) {
+      await fs.rm("tmp", { recursive: true });
+    }
+    writer = await createWriter("tmp");
+  });
+
+  afterEach(async () => {
+    if (existsSync("tmp")) {
+      await fs.rm("tmp", { recursive: true });
+    }
+  });
+
+  async function readJson(path: string) {
+    const content = await fs.readFile(path, "utf-8");
+    return JSON.parse(content);
+  }
+
+  it("should write data files", async () => {
+    const collections = [
+      {
+        name: "test",
+        documents: [
+          {
+            document: "one",
+          },
+          {
+            document: "two",
+          },
+          {
+            document: "three",
+          },
+        ],
+      },
+      {
+        name: "sample",
+        documents: [
+          {
+            document: "four",
+          },
+          {
+            document: "five",
+          },
+          {
+            document: "six",
+          },
+        ],
+      },
+    ];
+
+    await writer.createDataFiles(collections);
+
+    const allTests = await readJson("tmp/allTests.json");
+    expect(allTests).toEqual(["one", "two", "three"]);
+
+    const allSamples = await readJson("tmp/allSamples.json");
+    expect(allSamples).toEqual(["four", "five", "six"]);
+  });
+
+  it("should write javascript file", async () => {
+    await fs.writeFile(
+      "tmp/allTests.json",
+      JSON.stringify(["one", "two", "three"])
+    );
+    await fs.writeFile(
+      "tmp/allSamples.json",
+      JSON.stringify(["four", "five", "six"])
+    );
+
+    const collections = [
+      {
+        name: "test",
+      },
+      {
+        name: "sample",
+      },
+    ];
+
+    await writer.createJavaScriptFile({ collections });
+
+    // @ts-ignore the file is generated before
+    const indexJs = await import("./tmp/index.js");
+
+    expect(indexJs.allTests).toEqual(["one", "two", "three"]);
+    expect(indexJs.allSamples).toEqual(["four", "five", "six"]);
+  });
+
+  it("should write type definition file", async () => {
+    const collections = [
+      {
+        name: "test",
+        typeName: "Test",
+      },
+      {
+        name: "sample",
+        typeName: "Sample",
+      },
+    ];
+
+    await writer.createTypeDefinitionFile({
+      collections,
+      path: "./tmp/config.ts",
+      generateTypes: true,
+    });
+
+    const content = await fs.readFile("tmp/index.d.ts", "utf-8");
+    expect(content).toContain("export type Test =");
+    expect(content).toContain("export type Sample =");
+  });
+
+  it("should not write type definition file", async () => {
+    const collections = [
+      {
+        name: "test",
+        typeName: "Test",
+      }
+    ];
+
+    await writer.createTypeDefinitionFile({
+      collections,
+      path: "./tmp/config.ts",
+      generateTypes: false,
+    });
+
+    expect(existsSync("tmp/index.d.ts")).toBe(false);
+  });
+});