@confect/cli 7.0.0 → 9.0.0-next.0

package/src/confect/codegen.ts

@@ -1,8 +1,19 @@
 import { Spec } from "@confect/core";
-import { DatabaseSchema } from "@confect/server";
+import * as DatabaseSchema from "@confect/server/DatabaseSchema";
 import { Command } from "@effect/cli";
 import { FileSystem, Path } from "@effect/platform";
-import { Effect, Match, Option } from "effect";
+import { Array, Effect, Either, HashSet, Match, Option, Ref } from "effect";
+import { fromBundlerError } from "../BuildError";
+import * as CodegenError from "../CodegenError";
+import {
+  MissingImplFileError,
+  MissingSchemaFileError,
+  MissingSpecFileError,
+  SchemaInvalidDefaultExportError,
+} from "../CodegenError";
+import { ConfectDirectory } from "../ConfectDirectory";
+import { ConvexDirectory } from "../ConvexDirectory";
+import * as FunctionPaths from "../FunctionPaths";
 import {
   logFileAdded,
   logFileModified,
@@ -10,66 +21,94 @@ import {
   logPending,
   logSuccess,
 } from "../log";
-import { ConfectDirectory } from "../ConfectDirectory";
-import { ConvexDirectory } from "../ConvexDirectory";
+import {
+  discoverLeafImplFiles,
+  discoverLeafSpecFiles,
+  implPathForSpec,
+  registeredFunctionsRelativePath,
+  specPathForImpl,
+  toLeafModule,
+  toNodeRegistryLeaf,
+  validateImpl,
+  validateSpec,
+  type LeafModule,
+} from "../LeafModule";
+import {
+  assemblyNodesFromLeaves,
+  partitionByRuntime,
+} from "../SpecAssemblyNode";
 import * as templates from "../templates";
+import * as Bundler from "../Bundler";
 import {
-  bundleAndImport,
   generateAuthConfig,
   generateCrons,
   generateFunctions,
   generateHttp,
   removePathExtension,
+  removePathIfExists,
+  toModuleImportPath,
+  touchConvexSchema,
   writeFileStringAndLog,
+  WriteTracker,
 } from "../utils";
 
-const getNodeSpecPath = Effect.gen(function* () {
-  const path = yield* Path.Path;
-  const confectDirectory = yield* ConfectDirectory.get;
-  return path.join(confectDirectory, "nodeSpec.ts");
-});
-
-const loadNodeSpec = Effect.gen(function* () {
-  const fs = yield* FileSystem.FileSystem;
-  const nodeSpecPath = yield* getNodeSpecPath;
-
-  if (!(yield* fs.exists(nodeSpecPath))) {
-    return Option.none<Spec.AnyWithPropsWithRuntime<"Node">>();
-  }
-
-  const nodeSpecModule = yield* bundleAndImport(nodeSpecPath);
-  const nodeSpec = nodeSpecModule.default;
-
-  if (!Spec.isNodeSpec(nodeSpec)) {
-    return yield* Effect.die("nodeSpec.ts does not export a valid Node Spec");
-  }
-
-  return Option.some(nodeSpec);
-});
+const GENERATED_SPEC_PATH = "_generated/spec.ts";
+const GENERATED_NODE_SPEC_PATH = "_generated/nodeSpec.ts";
+
+const LEGACY_PATHS = [
+  "spec.ts",
+  "nodeSpec.ts",
+  "impl.ts",
+  "nodeImpl.ts",
+  "notesAndRandom.impl.ts",
+  "groups.impl.ts",
+  "_generated/registeredFunctions.ts",
+  "_generated/nodeRegisteredFunctions.ts",
+  "_generated/impl.ts",
+  "_generated/nodeImpl.ts",
+];
 
 export const codegen = Command.make("codegen", {}, () =>
   Effect.gen(function* () {
     yield* logPending("Performing initial sync…");
-    yield* codegenHandler;
-    yield* logSuccess("Generated files are up-to-date");
+    yield* codegenHandler.pipe(
+      Effect.asVoid,
+      Effect.tap(() => logSuccess("Generated files are up-to-date")),
+      CodegenError.tapAndLog,
+    );
   }),
 ).pipe(
   Command.withDescription(
-    "Generate `confect/_generated` files and the contents of the `convex` directory (except `tsconfig.json`)",
+    "Generate `confect/_generated` files and the contents of the `convex` directory (except `convex.config.ts` and `tsconfig.json`)",
   ),
 );
 
 export const codegenHandler = Effect.gen(function* () {
+  const tracker = yield* Ref.make(false);
+
+  const functionPaths = yield* runCodegen.pipe(
+    Effect.provideService(WriteTracker, tracker),
+  );
+
+  const anyWritesHappened = yield* Ref.get(tracker);
+  return { functionPaths, anyWritesHappened };
+});
+
+const runCodegen = Effect.gen(function* () {
   yield* generateConfectGeneratedDirectory;
+  // Validate schema first so its missing-file / invalid-default-export
+  // diagnostics surface ahead of impl bundling, which transitively depends
+  // on schema via `_generated/api.ts` and would otherwise blow up with a
+  // less actionable bundler error.
+  yield* validateSchema;
+  const leaves = yield* loadAndValidateLeafModules;
+  yield* removeLegacyFiles;
+  yield* generateAssembledSpecs(leaves);
+  yield* validateImplModules(leaves);
+  yield* generateGroupRegisteredFunctions(leaves);
+  yield* removeObsoleteRegisteredFunctions(leaves);
   yield* Effect.all(
-    [
-      generateApi,
-      generateRefs,
-      generateRegisteredFunctions,
-      generateNodeApi,
-      generateNodeRegisteredFunctions,
-      generateServices,
-    ],
+    [generateApi, generateRefs, generateNodeApi, generateServices],
     { concurrency: "unbounded" },
   );
   const [functionPaths] = yield* Effect.all(
@@ -82,6 +121,7 @@ export const codegenHandler = Effect.gen(function* () {
     ],
     { concurrency: "unbounded" },
   );
+  yield* touchConvexSchema;
   return functionPaths;
 });
 
@@ -98,6 +138,273 @@ const generateConfectGeneratedDirectory = Effect.gen(function* () {
   }
 });
 
+const loadAndValidateLeafModules = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const confectDirectory = yield* ConfectDirectory.get;
+  const specFiles = yield* discoverLeafSpecFiles;
+
+  const leaves = yield* Effect.forEach(specFiles, (specRelativePath) =>
+    Effect.gen(function* () {
+      const leaf = yield* toLeafModule(specRelativePath);
+      yield* validateSpec(leaf);
+
+      const implRelativePath = yield* implPathForSpec(specRelativePath);
+      const implAbsolutePath = path.join(confectDirectory, implRelativePath);
+      if (!(yield* fs.exists(implAbsolutePath))) {
+        return yield* new MissingImplFileError({
+          specPath: specRelativePath,
+          expectedImplPath: implRelativePath,
+        });
+      }
+
+      return leaf;
+    }),
+  );
+
+  yield* validateOrphanImpls(specFiles);
+
+  return leaves;
+});
+
+const validateOrphanImpls = (specFiles: ReadonlyArray<string>) =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const confectDirectory = yield* ConfectDirectory.get;
+    const implFiles = yield* discoverLeafImplFiles;
+    const specPaths = new Set(specFiles);
+
+    yield* Effect.forEach(implFiles, (implRelativePath) =>
+      Effect.gen(function* () {
+        const specRelativePath = yield* specPathForImpl(implRelativePath);
+        if (specPaths.has(specRelativePath)) {
+          return;
+        }
+
+        const specAbsolutePath = path.join(confectDirectory, specRelativePath);
+        if (!(yield* fs.exists(specAbsolutePath))) {
+          return yield* new MissingSpecFileError({
+            implPath: implRelativePath,
+            expectedSpecPath: specRelativePath,
+          });
+        }
+      }),
+    );
+  });
+
+const removeLegacyFiles = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const confectDirectory = yield* ConfectDirectory.get;
+
+  yield* Effect.forEach(LEGACY_PATHS, (relativePath) =>
+    Effect.gen(function* () {
+      const absolutePath = path.join(confectDirectory, relativePath);
+      if (yield* fs.exists(absolutePath)) {
+        yield* removePathIfExists(absolutePath);
+        yield* logFileRemoved(absolutePath);
+      }
+    }),
+  );
+});
+
+const generateAssembledSpecs = (leaves: ReadonlyArray<LeafModule>) =>
+  Effect.gen(function* () {
+    const path = yield* Path.Path;
+    const confectDirectory = yield* ConfectDirectory.get;
+    const { convex, node } = partitionByRuntime(leaves);
+
+    if (convex.length > 0) {
+      const nodes = assemblyNodesFromLeaves(convex);
+      const specContents = yield* templates.assembledSpec({
+        nodes,
+        runtime: "Convex",
+      });
+      yield* writeFileStringAndLog(
+        path.join(confectDirectory, GENERATED_SPEC_PATH),
+        specContents,
+      );
+    }
+
+    if (node.length > 0) {
+      const nodes = assemblyNodesFromLeaves(
+        Array.map(node, toNodeRegistryLeaf),
+      );
+      const nodeSpecContents = yield* templates.assembledSpec({
+        nodes,
+        runtime: "Node",
+      });
+      yield* writeFileStringAndLog(
+        path.join(confectDirectory, GENERATED_NODE_SPEC_PATH),
+        nodeSpecContents,
+      );
+    }
+  });
+
+const validateImplModules = (leaves: ReadonlyArray<LeafModule>) =>
+  Effect.forEach(leaves, validateImpl);
+
+const generateGroupRegisteredFunctions = (leaves: ReadonlyArray<LeafModule>) =>
+  Effect.gen(function* () {
+    const path = yield* Path.Path;
+    const confectDirectory = yield* ConfectDirectory.get;
+
+    yield* Effect.forEach(leaves, (leaf) =>
+      Effect.gen(function* () {
+        const registryRelativePath =
+          yield* registeredFunctionsRelativePath(leaf);
+        const registryPath = path.join(
+          confectDirectory,
+          "_generated",
+          registryRelativePath,
+        );
+        const registryDir = path.dirname(registryPath);
+        const fs = yield* FileSystem.FileSystem;
+        if (!(yield* fs.exists(registryDir))) {
+          yield* fs.makeDirectory(registryDir, { recursive: true });
+        }
+
+        const implRelativePath = yield* implPathForSpec(leaf.relativePath);
+        const apiFileName = leaf.runtime === "Node" ? "nodeApi.ts" : "api.ts";
+        const apiImportPath = yield* toModuleImportPath(
+          path.relative(
+            path.dirname(registryPath),
+            path.join(confectDirectory, "_generated", apiFileName),
+          ),
+        );
+        const implImportPath = yield* toModuleImportPath(
+          path.relative(
+            path.dirname(registryPath),
+            path.join(confectDirectory, implRelativePath),
+          ),
+        );
+
+        const contents = yield* templates.registeredFunctionsForGroup({
+          apiImportPath,
+          groupPathDot: leaf.registryGroupPathDot,
+          implImportPath,
+          layerExportName: leaf.exportName,
+          useNode: leaf.runtime === "Node",
+        });
+
+        yield* writeFileStringAndLog(registryPath, contents);
+      }),
+    );
+  });
+
+const removeObsoleteRegisteredFunctions = (leaves: ReadonlyArray<LeafModule>) =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const confectDirectory = yield* ConfectDirectory.get;
+    const registryRoot = path.join(
+      confectDirectory,
+      "_generated",
+      "registeredFunctions",
+    );
+
+    if (!(yield* fs.exists(registryRoot))) {
+      return;
+    }
+
+    const expected = new Set(
+      yield* Effect.forEach(leaves, (leaf) =>
+        registeredFunctionsRelativePath(leaf),
+      ),
+    );
+
+    const existing = yield* fs.readDirectory(registryRoot, { recursive: true });
+    yield* Effect.forEach(existing, (relativePath) => {
+      if (path.extname(relativePath) !== ".ts") {
+        return Effect.void;
+      }
+      const normalized = path.join("registeredFunctions", relativePath);
+      if (!expected.has(normalized)) {
+        return Effect.gen(function* () {
+          const absolutePath = path.join(registryRoot, relativePath);
+          if (yield* fs.exists(absolutePath)) {
+            yield* removePathIfExists(absolutePath);
+            yield* logFileRemoved(absolutePath);
+          }
+        });
+      }
+      return Effect.void;
+    });
+  });
+
+const getGeneratedSpecPath = Effect.gen(function* () {
+  const path = yield* Path.Path;
+  const confectDirectory = yield* ConfectDirectory.get;
+  return path.join(confectDirectory, GENERATED_SPEC_PATH);
+});
+
+const getGeneratedNodeSpecPath = Effect.gen(function* () {
+  const path = yield* Path.Path;
+  const confectDirectory = yield* ConfectDirectory.get;
+  return path.join(confectDirectory, GENERATED_NODE_SPEC_PATH);
+});
+
+const loadGeneratedSpec = Effect.gen(function* () {
+  const specPath = yield* getGeneratedSpecPath;
+  const { module: specModule } = yield* Bundler.bundle(specPath);
+  const spec = specModule.default;
+
+  if (!Spec.isConvexSpec(spec)) {
+    return yield* Effect.dieMessage(
+      "_generated/spec.ts does not export a valid Convex Spec",
+    );
+  }
+
+  return spec;
+});
+
+const loadGeneratedNodeSpec = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const nodeSpecPath = yield* getGeneratedNodeSpecPath;
+
+  if (!(yield* fs.exists(nodeSpecPath))) {
+    return Option.none<Spec.AnyWithPropsWithRuntime<"Node">>();
+  }
+
+  const { module: nodeSpecModule } = yield* Bundler.bundle(nodeSpecPath);
+  const nodeSpec = nodeSpecModule.default;
+
+  if (!Spec.isNodeSpec(nodeSpec)) {
+    return yield* Effect.dieMessage(
+      "_generated/nodeSpec.ts does not export a valid Node Spec",
+    );
+  }
+
+  return Option.some(nodeSpec);
+});
+
+const emptyFunctionPaths = FunctionPaths.FunctionPaths.make(HashSet.empty());
+
+export const loadPreviousFunctionPaths = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
+  const specPath = yield* getGeneratedSpecPath;
+
+  if (!(yield* fs.exists(specPath))) {
+    return emptyFunctionPaths;
+  }
+
+  const specEither = yield* loadGeneratedSpec.pipe(Effect.either);
+
+  return yield* Either.match(specEither, {
+    onLeft: () => Effect.succeed(emptyFunctionPaths),
+    onRight: (spec) =>
+      Effect.gen(function* () {
+        const nodeSpecOption = yield* loadGeneratedNodeSpec;
+        const mergedSpec = Option.match(nodeSpecOption, {
+          onNone: () => spec,
+          onSome: (nodeSpec) => Spec.merge(spec, nodeSpec),
+        });
+        return FunctionPaths.make(mergedSpec);
+      }),
+  });
+});
+
 const generateApi = Effect.gen(function* () {
   const path = yield* Path.Path;
   const confectDirectory = yield* ConfectDirectory.get;
@@ -105,12 +412,12 @@ const generateApi = Effect.gen(function* () {
   const apiPath = path.join(confectDirectory, "_generated", "api.ts");
   const apiDir = path.dirname(apiPath);
 
-  const schemaImportPath = yield* removePathExtension(
+  const schemaImportPath = yield* toModuleImportPath(
     path.relative(apiDir, path.join(confectDirectory, "schema.ts")),
   );
 
-  const specImportPath = yield* removePathExtension(
-    path.relative(apiDir, path.join(confectDirectory, "spec.ts")),
+  const specImportPath = yield* toModuleImportPath(
+    path.relative(apiDir, path.join(confectDirectory, GENERATED_SPEC_PATH)),
   );
 
   const apiContents = yield* templates.api({
@@ -126,12 +433,12 @@ export const generateNodeApi = Effect.gen(function* () {
   const path = yield* Path.Path;
   const confectDirectory = yield* ConfectDirectory.get;
 
-  const nodeSpecPath = yield* getNodeSpecPath;
+  const nodeSpecPath = yield* getGeneratedNodeSpecPath;
   const nodeApiPath = path.join(confectDirectory, "_generated", "nodeApi.ts");
 
   if (!(yield* fs.exists(nodeSpecPath))) {
     if (yield* fs.exists(nodeApiPath)) {
-      yield* fs.remove(nodeApiPath);
+      yield* removePathIfExists(nodeApiPath);
       yield* logFileRemoved(nodeApiPath);
     }
     return;
@@ -139,11 +446,11 @@ export const generateNodeApi = Effect.gen(function* () {
 
   const nodeApiDir = path.dirname(nodeApiPath);
 
-  const schemaImportPath = yield* removePathExtension(
+  const schemaImportPath = yield* toModuleImportPath(
     path.relative(nodeApiDir, path.join(confectDirectory, "schema.ts")),
   );
 
-  const nodeSpecImportPath = yield* removePathExtension(
+  const nodeSpecImportPath = yield* toModuleImportPath(
     path.relative(nodeApiDir, nodeSpecPath),
   );
 
@@ -156,47 +463,52 @@ export const generateNodeApi = Effect.gen(function* () {
 });
 
 const generateFunctionModules = Effect.gen(function* () {
-  const fs = yield* FileSystem.FileSystem;
-  const path = yield* Path.Path;
-  const confectDirectory = yield* ConfectDirectory.get;
-
-  const specPath = path.join(confectDirectory, "spec.ts");
-
-  const specModule = yield* bundleAndImport(specPath);
-  const spec = specModule.default;
-
-  if (!Spec.isConvexSpec(spec)) {
-    return yield* Effect.die("spec.ts does not export a valid Convex Spec");
-  }
-
-  const nodeImplPath = path.join(confectDirectory, "nodeImpl.ts");
-  const nodeImplExists = yield* fs.exists(nodeImplPath);
-  const nodeSpecOption = yield* loadNodeSpec;
+  const spec = yield* loadGeneratedSpec;
+  const nodeSpecOption = yield* loadGeneratedNodeSpec;
 
   const mergedSpec = Option.match(nodeSpecOption, {
     onNone: () => spec,
-    onSome: (nodeSpec) => (nodeImplExists ? Spec.merge(spec, nodeSpec) : spec),
+    onSome: (nodeSpec) => Spec.merge(spec, nodeSpec),
   });
 
   return yield* generateFunctions(mergedSpec);
 });
 
-const generateSchema = Effect.gen(function* () {
+export const validateSchema = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem;
   const path = yield* Path.Path;
   const confectDirectory = yield* ConfectDirectory.get;
-  const convexDirectory = yield* ConvexDirectory.get;
-
   const confectSchemaPath = path.join(confectDirectory, "schema.ts");
 
-  yield* bundleAndImport(confectSchemaPath).pipe(
-    Effect.andThen((schemaModule) => {
+  if (!(yield* fs.exists(confectSchemaPath))) {
+    return yield* new MissingSchemaFileError({ schemaPath: "schema.ts" });
+  }
+
+  yield* Bundler.bundle(confectSchemaPath).pipe(
+    Effect.mapError((error) => fromBundlerError("schema.ts", error)),
+    Effect.andThen(({ module: schemaModule }) => {
       const defaultExport = schemaModule.default;
 
-      return DatabaseSchema.isSchema(defaultExport)
+      return DatabaseSchema.isDatabaseSchema(defaultExport)
         ? Effect.succeed(defaultExport)
-        : Effect.die("Invalid schema module");
+        : Effect.fail(
+            new SchemaInvalidDefaultExportError({
+              schemaPath: "schema.ts",
+            }),
+          );
     }),
   );
+});
+
+const generateSchema = Effect.gen(function* () {
+  const path = yield* Path.Path;
+  const confectDirectory = yield* ConfectDirectory.get;
+  const convexDirectory = yield* ConvexDirectory.get;
+
+  const confectSchemaPath = path.join(confectDirectory, "schema.ts");
+
+  // `validateSchema` runs once at the top of `runCodegen`; no need to
+  // bundle the schema again here.
 
   const convexSchemaPath = path.join(convexDirectory, "schema.ts");
 
@@ -231,72 +543,6 @@ const generateServices = Effect.gen(function* () {
   yield* writeFileStringAndLog(servicesPath, servicesContentsString);
 });
 
-const generateRegisteredFunctions = Effect.gen(function* () {
-  const path = yield* Path.Path;
-  const confectDirectory = yield* ConfectDirectory.get;
-
-  const confectGeneratedDirectory = path.join(confectDirectory, "_generated");
-
-  const registeredFunctionsPath = path.join(
-    confectGeneratedDirectory,
-    "registeredFunctions.ts",
-  );
-  const implImportPath = yield* removePathExtension(
-    path.relative(
-      path.dirname(registeredFunctionsPath),
-      path.join(confectDirectory, "impl.ts"),
-    ),
-  );
-
-  const registeredFunctionsContents = yield* templates.registeredFunctions({
-    implImportPath,
-  });
-
-  yield* writeFileStringAndLog(
-    registeredFunctionsPath,
-    registeredFunctionsContents,
-  );
-});
-
-export const generateNodeRegisteredFunctions = Effect.gen(function* () {
-  const fs = yield* FileSystem.FileSystem;
-  const path = yield* Path.Path;
-  const confectDirectory = yield* ConfectDirectory.get;
-
-  const nodeImplPath = path.join(confectDirectory, "nodeImpl.ts");
-  const nodeSpecPath = yield* getNodeSpecPath;
-  const nodeRegisteredFunctionsPath = path.join(
-    confectDirectory,
-    "_generated",
-    "nodeRegisteredFunctions.ts",
-  );
-
-  const nodeImplExists = yield* fs.exists(nodeImplPath);
-  const nodeSpecExists = yield* fs.exists(nodeSpecPath);
-
-  if (!nodeImplExists || !nodeSpecExists) {
-    if (yield* fs.exists(nodeRegisteredFunctionsPath)) {
-      yield* fs.remove(nodeRegisteredFunctionsPath);
-      yield* logFileRemoved(nodeRegisteredFunctionsPath);
-    }
-    return;
-  }
-
-  const nodeImplImportPath = yield* removePathExtension(
-    path.relative(path.dirname(nodeRegisteredFunctionsPath), nodeImplPath),
-  );
-
-  const nodeRegisteredFunctionsContents =
-    yield* templates.nodeRegisteredFunctions({
-      nodeImplImportPath,
-    });
-
-  yield* writeFileStringAndLog(
-    nodeRegisteredFunctionsPath,
-    nodeRegisteredFunctionsContents,
-  );
-});
-
 const generateRefs = Effect.gen(function* () {
   const fs = yield* FileSystem.FileSystem;
   const path = yield* Path.Path;
@@ -306,19 +552,21 @@ const generateRefs = Effect.gen(function* () {
   const refsPath = path.join(confectGeneratedDirectory, "refs.ts");
   const refsDir = path.dirname(refsPath);
 
-  const specImportPath = yield* removePathExtension(
-    path.relative(refsDir, path.join(confectDirectory, "spec.ts")),
+  const specImportPath = yield* toModuleImportPath(
+    path.relative(refsDir, path.join(confectDirectory, GENERATED_SPEC_PATH)),
   );
 
-  const nodeSpecPath = yield* getNodeSpecPath;
+  const nodeSpecPath = yield* getGeneratedNodeSpecPath;
   const nodeSpecExists = yield* fs.exists(nodeSpecPath);
   const nodeSpecImportPath = nodeSpecExists
-    ? yield* removePathExtension(path.relative(refsDir, nodeSpecPath))
-    : null;
+    ? Option.some(
+        yield* toModuleImportPath(path.relative(refsDir, nodeSpecPath)),
+      )
+    : Option.none<string>();
 
   const refsContents = yield* templates.refs({
     specImportPath,
-    ...(nodeSpecImportPath === null ? {} : { nodeSpecImportPath }),
+    nodeSpecImportPath,
   });
 
   yield* writeFileStringAndLog(refsPath, refsContents);