@content-collections/core 0.1.0

package/dist/index.js

@@ -0,0 +1,630 @@
+// src/config.ts
+import { z } from "zod";
+
+// src/utils.ts
+import camelcase from "camelcase";
+import pluralize from "pluralize";
+function generateTypeName(name) {
+  const singularName = pluralize.singular(name);
+  return camelcase(singularName, { pascalCase: true });
+}
+function isDefined(value) {
+  return value !== void 0 && value !== null;
+}
+
+// src/config.ts
+function defineCollection(collection) {
+  let typeName = collection.typeName;
+  if (!typeName) {
+    typeName = generateTypeName(collection.name);
+  }
+  return {
+    ...collection,
+    typeName,
+    schema: collection.schema(z)
+  };
+}
+function defineConfig(config) {
+  return config;
+}
+
+// src/configurationReader.ts
+import * as esbuild from "esbuild";
+import fs from "fs/promises";
+import path from "path";
+
+// package.json
+var package_default = {
+  name: "@content-collections/core",
+  version: "0.1.0",
+  type: "module",
+  main: "dist/index.cjs",
+  types: "./dist/index.d.ts",
+  exports: {
+    "./package.json": "./package.json",
+    ".": {
+      import: "./dist/index.js",
+      types: "./dist/index.d.ts"
+    }
+  },
+  scripts: {
+    build: "tsup src/index.ts --format esm --dts -d dist",
+    typecheck: "tsc",
+    test: "vitest --run --coverage"
+  },
+  devDependencies: {
+    "@types/micromatch": "^4.0.6",
+    "@types/node": "^20.9.0",
+    "@types/pluralize": "^0.0.33",
+    "@vitest/coverage-v8": "^1.1.0",
+    tsup: "^7.2.0",
+    tsx: "^4.1.1",
+    typescript: "^5.3.3",
+    vitest: "^1.1.0"
+  },
+  dependencies: {
+    "@parcel/watcher": "^2.3.0",
+    camelcase: "^8.0.0",
+    esbuild: "^0.19.5",
+    "fast-glob": "^3.3.2",
+    "gray-matter": "^4.0.3",
+    micromatch: "^4.0.5",
+    pluralize: "^8.0.0",
+    zod: "^3.22.4"
+  }
+};
+
+// src/configurationReader.ts
+import { existsSync } from "fs";
+var ConfigurationError = class extends Error {
+  type;
+  constructor(type, message) {
+    super(message);
+    this.type = type;
+  }
+};
+var importPathPlugin = {
+  name: "import-path",
+  setup(build2) {
+    build2.onResolve({ filter: /^\@content-collections\/core$/ }, () => {
+      return { path: path.join(__dirname, "index.ts"), external: true };
+    });
+  }
+};
+var defaultConfigName = "content-collection-config.mjs";
+function resolveCacheDir(config, options) {
+  if (options.cacheDir) {
+    return options.cacheDir;
+  }
+  return path.join(path.dirname(config), ".content-collections", "cache");
+}
+async function compile(configurationPath, outfile) {
+  const plugins = [];
+  if (process.env.NODE_ENV === "test") {
+    plugins.push(importPathPlugin);
+  }
+  await esbuild.build({
+    entryPoints: [configurationPath],
+    external: [...Object.keys(package_default.dependencies), "@content-collections/*"],
+    bundle: true,
+    platform: "node",
+    format: "esm",
+    plugins,
+    outfile
+  });
+}
+function createConfigurationReader() {
+  return async (configurationPath, options = {
+    configName: defaultConfigName
+  }) => {
+    if (!existsSync(configurationPath)) {
+      throw new ConfigurationError(
+        "Read",
+        `configuration file ${configurationPath} does not exist`
+      );
+    }
+    const cacheDir = resolveCacheDir(configurationPath, options);
+    await fs.mkdir(cacheDir, { recursive: true });
+    const outfile = path.join(cacheDir, options.configName);
+    try {
+      await compile(configurationPath, outfile);
+    } catch (error) {
+      throw new ConfigurationError(
+        "Compile",
+        `configuration file ${configurationPath} is invalid: ${error}`
+      );
+    }
+    const module = await import(`file://${path.resolve(outfile)}?x=${Date.now()}`);
+    return {
+      ...module.default,
+      path: configurationPath,
+      generateTypes: true
+    };
+  };
+}
+
+// src/collector.ts
+import matter from "gray-matter";
+import fg from "fast-glob";
+import { readFile } from "fs/promises";
+import path2 from "path";
+var CollectError = class extends Error {
+  type;
+  constructor(type, message) {
+    super(message);
+    this.type = type;
+  }
+};
+function createCollector(emitter, baseDirectory = ".") {
+  async function read(filePath) {
+    try {
+      return await readFile(filePath, "utf-8");
+    } catch (error) {
+      emitter.emit("collector:read-error", {
+        filePath,
+        error: new CollectError("Read", String(error))
+      });
+      return null;
+    }
+  }
+  function parse(file) {
+    const { data, content } = matter(file);
+    return {
+      ...data,
+      content
+    };
+  }
+  async function collectFile(directory, filePath) {
+    const file = await read(path2.join(directory, filePath));
+    if (!file) {
+      return null;
+    }
+    try {
+      const data = parse(file);
+      return {
+        data,
+        path: filePath
+      };
+    } catch (error) {
+      emitter.emit("collector:parse-error", {
+        filePath: path2.join(directory, filePath),
+        error: new CollectError("Parse", String(error))
+      });
+      return null;
+    }
+  }
+  async function resolveCollection(collection) {
+    const collectionDirectory = path2.join(baseDirectory, collection.directory);
+    const filePaths = await fg(collection.include, {
+      cwd: collectionDirectory,
+      onlyFiles: true,
+      absolute: false
+    });
+    const promises = filePaths.map(
+      (filePath) => collectFile(collectionDirectory, filePath)
+    );
+    const files = await Promise.all(promises);
+    return {
+      ...collection,
+      files: files.filter(isDefined)
+    };
+  }
+  async function collect(unresolvedCollections) {
+    const promises = unresolvedCollections.map(
+      (collection) => resolveCollection(collection)
+    );
+    return await Promise.all(promises);
+  }
+  return {
+    collect,
+    collectFile
+  };
+}
+
+// src/writer.ts
+import fs2 from "fs/promises";
+import path3 from "path";
+import pluralize2 from "pluralize";
+function createArrayConstName(name) {
+  let suffix = name.charAt(0).toUpperCase() + name.slice(1);
+  return "all" + pluralize2(suffix);
+}
+async function createDataFile(directory, collection) {
+  const dataPath = path3.join(
+    directory,
+    `${createArrayConstName(collection.name)}.json`
+  );
+  await fs2.writeFile(
+    dataPath,
+    JSON.stringify(
+      collection.documents.map((doc) => doc.document),
+      null,
+      2
+    )
+  );
+}
+function createDataFiles(directory, collections) {
+  return Promise.all(
+    collections.map((collection) => createDataFile(directory, collection))
+  );
+}
+async function createJavaScriptFile(directory, configuration) {
+  const collections = configuration.collections.map(
+    ({ name }) => createArrayConstName(name)
+  );
+  let content = `// generated by content-collections at ${/* @__PURE__ */ new Date()}
+
+`;
+  for (const name of collections) {
+    content += `import ${name} from "./${name}.json";
+`;
+  }
+  content += "\n";
+  content += "export { " + collections.join(", ") + " };\n";
+  await fs2.writeFile(path3.join(directory, "index.js"), content, "utf-8");
+}
+async function createTypeDefinitionFile(directory, configuration) {
+  if (!configuration.generateTypes) {
+    return;
+  }
+  const importPath = path3.relative(directory, configuration.path);
+  let content = `import configuration from "${importPath}";
+import { GetTypeByName } from "@content-collections/core";
+`;
+  const collections = configuration.collections;
+  for (const collection of collections) {
+    content += `
+`;
+    content += `export type ${collection.typeName} = GetTypeByName<typeof configuration, "${collection.name}">;
+`;
+    content += `export declare const ${createArrayConstName(
+      collection.name
+    )}: Array<${collection.typeName}>;
+`;
+  }
+  content += "\n";
+  content += "export {};\n";
+  await fs2.writeFile(path3.join(directory, "index.d.ts"), content, "utf-8");
+}
+async function createWriter(directory) {
+  await fs2.mkdir(directory, { recursive: true });
+  return {
+    createJavaScriptFile: (configuration) => createJavaScriptFile(directory, configuration),
+    createTypeDefinitionFile: (configuration) => createTypeDefinitionFile(directory, configuration),
+    createDataFiles: (collections) => createDataFiles(directory, collections)
+  };
+}
+
+// src/transformer.ts
+import { basename, dirname, extname } from "path";
+import { z as z2 } from "zod";
+var TransformError = class extends Error {
+  type;
+  constructor(type, message) {
+    super(message);
+    this.type = type;
+  }
+};
+function createPath(path6, ext) {
+  let p = path6.slice(0, -ext.length);
+  if (p.endsWith("/index")) {
+    p = p.slice(0, -6);
+  }
+  return p;
+}
+function createTransformer(emitter) {
+  function createSchema(schema) {
+    return z2.object({
+      content: z2.string(),
+      ...schema
+    });
+  }
+  async function parseFile(collection, file) {
+    const { data, path: path6 } = file;
+    const schema = createSchema(collection.schema);
+    let parsedData = await schema.safeParseAsync(data);
+    if (!parsedData.success) {
+      emitter.emit("transformer:validation-error", {
+        collection,
+        file,
+        error: new TransformError("Validation", parsedData.error.message)
+      });
+      return null;
+    }
+    const ext = extname(path6);
+    let extension = ext;
+    if (extension.startsWith(".")) {
+      extension = extension.slice(1);
+    }
+    const document = {
+      ...parsedData.data,
+      _meta: {
+        filePath: path6,
+        fileName: basename(path6),
+        directory: dirname(path6),
+        extension,
+        path: createPath(path6, ext)
+      }
+    };
+    return {
+      document
+    };
+  }
+  async function parseCollection(collection) {
+    const promises = collection.files.map(
+      (file) => parseFile(collection, file)
+    );
+    return {
+      ...collection,
+      documents: (await Promise.all(promises)).filter(isDefined)
+    };
+  }
+  function createContext(collections) {
+    return {
+      documents: (collection) => {
+        const resolved = collections.find((c) => c.name === collection.name);
+        if (!resolved) {
+          throw new TransformError(
+            "Configuration",
+            `Collection ${collection.name} not found, do you have registered it in your configuration?`
+          );
+        }
+        return resolved.documents.map((doc) => doc.document);
+      }
+    };
+  }
+  async function transformCollection(collections, collection) {
+    if (collection.transform) {
+      const docs = [];
+      const context = createContext(collections);
+      for (const doc of collection.documents) {
+        try {
+          docs.push({
+            ...doc,
+            document: await collection.transform(context, doc.document)
+          });
+        } catch (error) {
+          if (error instanceof TransformError) {
+            emitter.emit("transformer:error", {
+              collection,
+              error
+            });
+          } else {
+            emitter.emit("transformer:error", {
+              collection,
+              error: new TransformError("Transform", String(error))
+            });
+          }
+        }
+      }
+      return docs;
+    }
+    return collection.documents;
+  }
+  return async (untransformedCollections) => {
+    const promises = untransformedCollections.map(
+      (collection) => parseCollection(collection)
+    );
+    const collections = await Promise.all(promises);
+    for (const collection of collections) {
+      collection.documents = await transformCollection(collections, collection);
+    }
+    return collections;
+  };
+}
+
+// src/synchronizer.ts
+import micromatch from "micromatch";
+import path4 from "path";
+function createSynchronizer(readCollectionFile, collections, baseDirectory = ".") {
+  function findCollection(filePath) {
+    const resolvedFilePath = path4.resolve(filePath);
+    return collections.find((collection) => {
+      return resolvedFilePath.startsWith(
+        path4.resolve(baseDirectory, collection.directory)
+      );
+    });
+  }
+  function createRelativePath(collectionPath, filePath) {
+    const resolvedCollectionPath = path4.resolve(baseDirectory, collectionPath);
+    const resolvedFilePath = path4.resolve(filePath);
+    let relativePath = resolvedFilePath.slice(resolvedCollectionPath.length);
+    if (relativePath.startsWith("/")) {
+      relativePath = relativePath.slice(1);
+    }
+    return relativePath;
+  }
+  function resolve(filePath) {
+    const collection = findCollection(filePath);
+    if (!collection) {
+      return null;
+    }
+    const relativePath = createRelativePath(collection.directory, filePath);
+    if (!micromatch.isMatch(relativePath, collection.include)) {
+      return null;
+    }
+    return {
+      collection,
+      relativePath
+    };
+  }
+  function deleted(filePath) {
+    const resolved = resolve(filePath);
+    if (!resolved) {
+      return false;
+    }
+    const { collection, relativePath } = resolved;
+    const index = collection.files.findIndex(
+      (file) => file.path === relativePath
+    );
+    const deleted2 = collection.files.splice(index, 1);
+    return deleted2.length > 0;
+  }
+  async function changed(filePath) {
+    const resolved = resolve(filePath);
+    if (!resolved) {
+      return false;
+    }
+    const { collection, relativePath } = resolved;
+    const index = collection.files.findIndex(
+      (file2) => file2.path === relativePath
+    );
+    const file = await readCollectionFile(
+      path4.join(baseDirectory, collection.directory),
+      relativePath
+    );
+    if (!file) {
+      return false;
+    }
+    if (index === -1) {
+      collection.files.push(file);
+    } else {
+      collection.files[index] = file;
+    }
+    return true;
+  }
+  return {
+    deleted,
+    changed
+  };
+}
+
+// src/builder.ts
+import path5 from "path";
+
+// src/watcher.ts
+import watcher from "@parcel/watcher";
+async function createWatcher(emitter, paths, sync, build2) {
+  const onChange = async (error, events) => {
+    if (error) {
+      console.error(error);
+      return;
+    }
+    let rebuild = false;
+    for (const event of events) {
+      if (await sync(event.type, event.path)) {
+        emitter.emit("watcher:file-changed", {
+          filePath: event.path,
+          modification: event.type
+        });
+        rebuild = true;
+      }
+    }
+    if (rebuild) {
+      await build2();
+    }
+  };
+  const subscriptions = await Promise.all(
+    paths.map((path6) => watcher.subscribe(path6, onChange))
+  );
+  return {
+    unsubscribe: async () => {
+      await Promise.all(
+        subscriptions.map((subscription) => subscription.unsubscribe())
+      );
+      return;
+    }
+  };
+}
+
+// src/events.ts
+import { EventEmitter } from "events";
+function isEventWithError(event) {
+  return typeof event === "object" && event !== null && "error" in event;
+}
+function createEmitter() {
+  const emitter = new EventEmitter();
+  function on(key, listener) {
+    emitter.on(key, listener);
+  }
+  function emit(key, event) {
+    emitter.emit(key, event);
+    if (isEventWithError(event)) {
+      emitter.emit("_error", {
+        ...event,
+        _event: key
+      });
+    }
+    emitter.emit("_all", {
+      ...event,
+      _event: key
+    });
+  }
+  return {
+    on,
+    emit
+  };
+}
+
+// src/builder.ts
+function resolveOutputDir(baseDirectory, options) {
+  if (options.outputDir) {
+    return options.outputDir;
+  }
+  return path5.join(baseDirectory, ".content-collections", "generated");
+}
+async function createBuilder(configurationPath, options = {
+  configName: defaultConfigName
+}) {
+  const emitter = createEmitter();
+  const readConfiguration = createConfigurationReader();
+  const configuration = await readConfiguration(configurationPath, options);
+  const baseDirectory = path5.dirname(configurationPath);
+  const directory = resolveOutputDir(baseDirectory, options);
+  const collector = createCollector(emitter, baseDirectory);
+  const writer = await createWriter(directory);
+  const [resolved] = await Promise.all([
+    collector.collect(configuration.collections),
+    writer.createJavaScriptFile(configuration),
+    writer.createTypeDefinitionFile(configuration)
+  ]);
+  const synchronizer = createSynchronizer(
+    collector.collectFile,
+    resolved,
+    baseDirectory
+  );
+  const transform = createTransformer(emitter);
+  async function sync(modification, filePath) {
+    if (modification === "delete") {
+      return synchronizer.deleted(filePath);
+    }
+    return synchronizer.changed(filePath);
+  }
+  async function build2() {
+    const startedAt = Date.now();
+    emitter.emit("builder:start", {
+      startedAt
+    });
+    const collections = await transform(resolved);
+    await writer.createDataFiles(collections);
+    const pendingOnSuccess = collections.filter((collection) => Boolean(collection.onSuccess)).map(
+      (collection) => collection.onSuccess?.(collection.documents.map((doc) => doc.document))
+    );
+    await Promise.all(pendingOnSuccess.filter(isDefined));
+    emitter.emit("builder:end", {
+      startedAt,
+      endedAt: Date.now()
+    });
+  }
+  async function watch() {
+    const paths = resolved.map(
+      (collection) => path5.join(baseDirectory, collection.directory)
+    );
+    const watcher2 = await createWatcher(emitter, paths, sync, build2);
+    return watcher2;
+  }
+  return {
+    sync,
+    build: build2,
+    watch,
+    on: emitter.on
+  };
+}
+export {
+  CollectError,
+  ConfigurationError,
+  TransformError,
+  createBuilder,
+  defineCollection,
+  defineConfig
+};

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@content-collections/core",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "dist/index.cjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "devDependencies": {
+    "@types/micromatch": "^4.0.6",
+    "@types/node": "^20.9.0",
+    "@types/pluralize": "^0.0.33",
+    "@vitest/coverage-v8": "^1.1.0",
+    "tsup": "^7.2.0",
+    "tsx": "^4.1.1",
+    "typescript": "^5.3.3",
+    "vitest": "^1.1.0"
+  },
+  "dependencies": {
+    "@parcel/watcher": "^2.3.0",
+    "camelcase": "^8.0.0",
+    "esbuild": "^0.19.5",
+    "fast-glob": "^3.3.2",
+    "gray-matter": "^4.0.3",
+    "micromatch": "^4.0.5",
+    "pluralize": "^8.0.0",
+    "zod": "^3.22.4"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts -d dist",
+    "typecheck": "tsc",
+    "test": "vitest --run --coverage"
+  }
+}

package/src/__tests__/.content-collections/cache/config.001.ts.js

@@ -0,0 +1,19 @@
+// src/__tests__/config.001.ts
+import { defineCollection, defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+var posts = defineCollection({
+  name: "posts",
+  typeName: "Post",
+  schema: (z) => ({
+    title: z.string().min(5),
+    description: z.string().min(10),
+    date: z.union([z.string().regex(/^\d{4}-\d{2}-\d{2}$/), z.date()]).transform((val) => new Date(val))
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?"
+});
+var config_001_default = defineConfig({
+  collections: [posts]
+});
+export {
+  config_001_default as default
+};

package/src/__tests__/.content-collections/cache/config.002.mjs

@@ -0,0 +1,16 @@
+// src/__tests__/config.002.ts
+import { defineCollection, defineConfig } from "/Users/sdorra/Projects/sdorra/content-collections/packages/core/src/index.ts";
+var posts = defineCollection({
+  name: "posts",
+  schema: (z) => ({
+    title: z.string()
+  }),
+  directory: "sources/posts",
+  include: "**/*.md(x)?"
+});
+var config_002_default = defineConfig({
+  collections: [posts]
+});
+export {
+  config_002_default as default
+};