alchemy 0.1.2 → 0.1.3

package/src/agent/module.ts

@@ -0,0 +1,418 @@
+import type { CoreMessage, LanguageModel } from "ai";
+import { createPatch } from "diff";
+import fs from "node:fs/promises";
+import { z } from "zod";
+import { Folder } from "../fs";
+import type { Input } from "../input";
+import { Design, type DesignInput } from "../markdown/design";
+import { Requirements, type RequirementsInput } from "../markdown/requirements";
+import {
+  type Context,
+  type CreateContext,
+  Resource,
+  type UpdateContext,
+} from "../resource";
+import { slugify } from "../slug";
+import { TypeScriptFile, type TypeScriptFileInput } from "../typescript";
+import type { PackageJson } from "../typescript/package";
+import type { TypeScriptConfig } from "../typescript/tsconfig";
+import { generateObject } from "./ai";
+import { dependenciesAsMessages } from "./dependencies";
+import type { FileContext } from "./file-context";
+import { type ModelId, resolveModel } from "./model";
+import { Prompts } from "./prompts";
+
+type ProjectFileType = z.infer<typeof ProjectFileType>;
+const ProjectFileType = z.enum([
+  "folder",
+  "design",
+  "typescript",
+  // "package.json",
+  // "tsconfig.json",
+  "css",
+  "scss",
+  "javascript",
+  "html",
+]);
+
+const upstream = z
+  .array(z.string())
+  .describe(
+    [
+      "A list of all this file's upstream dependencies (files and folders).",
+      "These files and folders must exist prior to creating or updating 'this' file.",
+      "'this' meaning the file described by the key in the object.",
+      "If the plan includes a type: folder to be created, and a file is contained",
+      "within that folder, then it must also be listed in the upstream dependencies.",
+      "Don't include files as upstream dependencies if that file's content is not relevant to this file.",
+    ].join(" "),
+  );
+
+interface CreateFile extends z.infer<typeof CreateFile> {}
+const CreateFile = z.object({
+  type: ProjectFileType,
+  upstream,
+  operation: z.enum(["create"]),
+});
+
+interface UpdateFile extends z.infer<typeof UpdateFile> {}
+const UpdateFile = z.object({
+  type: ProjectFileType,
+  upstream,
+  operation: z
+    .enum(["create", "update"])
+    .describe(
+      [
+        "The operation to perform on this fle (one of: create, update, delete, noop).",
+        "If the document describes a new file that needs to be created, the operation is 'create'.",
+        "If the document's content has changed such that a file that was previously created must be updated, the operation is 'update'.",
+        "If the document's content has changed, but not in a way that is relevant to this file, the operation is 'noop'.",
+        "If a file that was previously created is no longer mentioned in the document, the operation is 'delete'.",
+      ].join(" "),
+    ),
+});
+
+export type CreatePlan = z.infer<typeof CreatePlan>;
+export const CreatePlan = z.object({
+  files: z.record(CreateFile),
+});
+
+export type UpdatePlan = z.infer<typeof UpdatePlan>;
+export const UpdatePlan = z.object({
+  files: z.record(z.union([UpdateFile, z.enum(["delete", "noop"])])),
+});
+
+type OutputFile =
+  | TypeScriptFile
+  | PackageJson
+  | TypeScriptConfig
+  | Folder
+  | Module;
+
+export interface ModuleOutput {
+  path: string;
+  content: string;
+  plan: CreatePlan | UpdatePlan;
+  files: Record<string, OutputFile>;
+}
+
+export interface ModuleInput {
+  path: string;
+  content: string;
+  model?: ModelId;
+  dependencies?: FileContext[];
+}
+
+export class Module extends Resource(
+  "Module",
+  async (
+    ctx: Context<ModuleOutput>,
+    props: ModuleInput,
+  ): Promise<ModuleOutput | void> => {
+    if (ctx.event === "delete") {
+      return;
+    }
+
+    const model = await resolveModel(props.model ?? "gpt-4o");
+
+    const plan: CreatePlan | UpdatePlan =
+      ctx.event === "create"
+        ? await createPlan(ctx, props, model)
+        : await updatePlan(ctx, props, model);
+
+    console.log(JSON.stringify(plan, null, 2));
+
+    const dependencies = Object.fromEntries(
+      Object.entries(plan.files).flatMap(([path, file]) =>
+        file === "delete" || file === "noop" ? [] : [[path, file.upstream]],
+      ),
+    );
+
+    const files: {
+      [path: string]: Input<FileContext>;
+    } = {};
+
+    for (const filePath of topoSort(dependencies)) {
+      const filePlan = plan.files[filePath];
+      if (filePlan !== "delete") {
+        const file = await createFile(
+          filePath,
+          filePlan === "noop"
+            ? "noop"
+            : (filePlan.operation as "create" | "update"),
+          filePlan === "noop" ? null : filePlan,
+          dependencies[filePath]?.map((dep) => files[dep]),
+        );
+        if (file) {
+          files[filePath] = file;
+        }
+      }
+    }
+
+    return {
+      path: props.path,
+      plan,
+      content: props.content,
+      // @ts-ignore - TODO: fix this
+      files,
+    };
+
+    async function createFile(
+      filePath: string,
+      op: "create" | "update" | "noop",
+      filePlan: CreateFile | UpdateFile | null,
+      dependencies: Input<FileContext>[],
+    ): Promise<Input<FileContext> | undefined> {
+      const slug = slugify(filePath);
+
+      // cache inputs for no-ops
+      async function inputs<T>(
+        key: string,
+        input: Input<T>,
+      ): Promise<Input<T>> {
+        return (op === "noop" ? await ctx.get<T>(key) : input) ?? input;
+      }
+
+      if (filePlan === null) {
+        // restore the file plan from the cache for no-ops
+        filePlan = (await ctx.get<CreateFile | UpdateFile>(filePath))!;
+      } else {
+        await ctx.set(filePath, filePlan);
+      }
+      if (filePlan.type === "design") {
+        const design = new Design(
+          `${slug}-design`,
+          await inputs<DesignInput>(`${slug}-design`, {
+            path: filePath,
+            prompt: [
+              "Context:",
+              props.content,
+              "",
+              `Produce the design: ${filePath}`,
+            ].join("\n"),
+            dependencies,
+            model: "o3-mini",
+          }),
+        );
+
+        const module = new Module(
+          slug,
+          await inputs<ModuleInput>(slug, {
+            model: props.model,
+            path: filePath,
+            content: design.content,
+            dependencies,
+          }),
+        );
+
+        return module;
+      } else if (filePlan.type === "typescript") {
+        const requirementsSlug = slug.replace(".ts", ".md");
+        const requirements = new Requirements(
+          requirementsSlug,
+          await inputs<RequirementsInput>(requirementsSlug, {
+            // file: file.path.replace(".ts", ".md").replace("src/", "design/"),
+            requirements: [
+              "Context:",
+              props.content,
+              "Define the low-level requirements for this TypeScript file:",
+              filePath,
+            ],
+            dependencies,
+          }),
+        );
+        return new TypeScriptFile(
+          slug,
+          await inputs<TypeScriptFileInput>(slug, {
+            path: filePath,
+            requirements: requirements.content,
+            dependencies,
+          }),
+        );
+        // } else if (filePlan.type === "package.json") {
+        //   return new PackageJson(
+        //     slugify(filePath),
+        //     await inputs<PackageJsonInput>(slug, {
+        //       path: filePath,
+        //       name: "todo",
+        //       requirements: [
+        //         "Context:",
+        //         props.content,
+        //         "Create the package.json:",
+        //         filePath,
+        //       ],
+        //       dependencies,
+        //     }),
+        //   );
+        // } else if (filePlan.type === "tsconfig.json") {
+        //   return new TypeScriptConfig(
+        //     slugify(filePath),
+        //     await inputs<TypeScriptConfigInput>(slug, {
+        //       path: filePath,
+        //       requirements: [props.content],
+        //       dependencies,
+        //     }),
+        //   );
+      } else if (filePlan.type === "folder") {
+        return new Folder(slug, filePath, ...dependencies.map((d) => d.path));
+      } else {
+        console.error(`Unknown file type: ${filePlan.type}`);
+        return undefined;
+      }
+    }
+  },
+) {}
+
+async function createPlan(
+  ctx: CreateContext,
+  props: ModuleInput,
+  model: LanguageModel,
+): Promise<CreatePlan> {
+  const result = await generateObject({
+    model,
+    schema: CreatePlan,
+    temperature: 0.1,
+    messages: [
+      {
+        role: "system",
+        content: Prompts.program,
+      },
+      ...excludeDependenciesFromPlan(props.dependencies),
+      {
+        role: "user",
+        content: [
+          "Derive the execution plan for this document:",
+          "```md",
+          props.content,
+          "```",
+          "You must only include files explicitly mentioned in the document.",
+          "Do not preemptively assume that any other files are needed.",
+        ].join("\n\n"),
+      },
+    ],
+  });
+
+  return result.object;
+}
+
+async function updatePlan(
+  ctx: UpdateContext<ModuleOutput>,
+  props: ModuleInput,
+  model: LanguageModel,
+): Promise<UpdatePlan> {
+  const newContent = await fs.readFile(props.path, "utf-8");
+  const oldContent = ctx.output.content;
+  const patch = createPatch(props.path, oldContent, newContent);
+  console.log({
+    role: "assistant",
+    content: JSON.stringify(ctx.output.plan),
+  });
+  try {
+    const result = await generateObject({
+      model,
+      schema: UpdatePlan,
+      temperature: 0.1,
+      messages: [
+        {
+          role: "system",
+          content: Prompts.program,
+        },
+        ...excludeDependenciesFromPlan(props.dependencies),
+        {
+          role: "user",
+          content: [
+            "Derive the execution plan for this document:",
+            "```md",
+            oldContent,
+            "```",
+          ].join("\n\n"),
+        } satisfies CoreMessage,
+        {
+          role: "assistant",
+          content: JSON.stringify(ctx.output.plan),
+        } satisfies CoreMessage,
+        {
+          role: "user",
+          content: [
+            "I've made the following changes to the document:",
+            "```diff",
+            patch,
+            "```",
+            "Please update the plan to reflect the changes.",
+            "You must only include files explicitly mentioned in the document.",
+            "Do not preemptively assume that any other files are needed.",
+          ].join("\n"),
+        } satisfies CoreMessage,
+      ],
+    });
+
+    return result.object;
+  } catch (error) {
+    console.log("Failed to update plan");
+    console.error(error);
+    throw error;
+  }
+}
+
+function excludeDependenciesFromPlan(dependencies: FileContext[] | undefined) {
+  if (dependencies) {
+    return [
+      ...dependenciesAsMessages(dependencies),
+      ...(dependencies
+        ? [
+            {
+              role: "user" as const,
+              content: [
+                "Remember that these files don't need to be create",
+                ...dependencies.map((dep) => `- ${dep.path}`),
+              ].join("\n"),
+            },
+            {
+              role: "assistant" as const,
+              content: [
+                "Got it, I acknowledge that the following files will not be included as create/update/delete/noop nodes in the plan",
+                ...dependencies.map((dep) => `- ${dep.path}`),
+              ].join("\n"),
+            },
+          ]
+        : []),
+    ];
+  } else {
+    return [];
+  }
+}
+
+function topoSort(dependencies: Record<string, string[]>): string[] {
+  const visited = new Set<string>();
+  const temp = new Set<string>();
+  const order: string[] = [];
+
+  function visit(file: string) {
+    if (temp.has(file)) {
+      throw new Error(`Circular dependency detected involving ${file}`);
+    }
+    if (visited.has(file)) {
+      return;
+    }
+
+    temp.add(file);
+
+    const deps = dependencies[file] || [];
+    for (const dep of deps) {
+      visit(dep);
+    }
+
+    temp.delete(file);
+    visited.add(file);
+    order.push(file);
+  }
+
+  const files = Object.keys(dependencies);
+  for (const file of files) {
+    if (!visited.has(file)) {
+      visit(file);
+    }
+  }
+
+  return order;
+}

package/src/{components/agent → agent}/openai.ts

@@ -1,4 +1,7 @@
+import { z } from "zod";
 export const OpenAIModels = [
+  "o3-mini",
+
   // O1 models
   "o1",
   "o1-2024-12-17",
@@ -33,12 +36,15 @@ export const OpenAIModels = [
   "gpt-3.5-turbo-1106",
 ] as const;
 
+export const OpenAIModel = z.enum(OpenAIModels);
+
 export type OpenAIChatModelId = (typeof OpenAIModels)[number] | (string & {});
 
 export function isOpenAIModel(model: string): model is OpenAIChatModelId {
   return (
     model.startsWith("gpt-") ||
     model.startsWith("o1") ||
+    model.startsWith("o3") ||
     model.startsWith("gpt-4o")
   );
 }

package/src/agent/program.md

@@ -0,0 +1,38 @@
+Alchemy is a markdown-based development framework.
+
+A markdown document can be thought of as a module file in a programming language.
+
+It consists of expressions that:
+
+1. describe system requirements
+2. enumerate and describe `1..*` files that should be created
+3. refer to `0..*` files that should be referenced to inform the creation of files
+
+Design documents are succinct and concise.
+We purposely avoid over documentation and over explanation.
+
+It should be thought more of a sequence of statements in a programming language.
+
+Example:
+
+```md
+# Backend Server
+
+The backend server is a Hono app that exposes a REST API.
+
+## API Reference
+
+- [GetUser](src/get-user.ts) - gets a user by UserID (ULID)
+- [CreateUser](src/create-user.ts)
+  - inserts a User (Email, FirstName, LastName)
+  - generates a UserID
+  - Email must be Unique
+
+## Database Schema
+
+Refer to the [Database Schema](src/schema.sql.ts)
+```
+
+Notice how the example document is very matter of a fact.
+Line, line, line.
+Simple, logical components capturing all the important details and nothing more.

package/src/agent/program.ts

@@ -0,0 +1,37 @@
+import fs from "node:fs/promises";
+import type { Context } from "../resource";
+import { Resource } from "../resource";
+import type { ModelId } from "./model";
+import { Module } from "./module";
+
+export class Program extends Resource(
+  "Program",
+  {
+    alwaysUpdate: true,
+  },
+  async (
+    ctx: Context<{
+      module: Module;
+    }>,
+    props: {
+      path: string;
+      model?: ModelId;
+    },
+  ) => {
+    if (ctx.event === "delete") {
+      return;
+    }
+
+    const content = await fs.readFile(props.path, "utf-8");
+
+    const module = new Module("module", {
+      path: props.path,
+      content,
+      model: props.model,
+    });
+
+    return {
+      module,
+    };
+  },
+) {}

package/src/agent/prompts.ts

@@ -0,0 +1,20 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+// TODO: replace with import.meta.glob
+// see: https://github.com/oven-sh/bun/issues/6060
+
+const __dirname = new URL(".", import.meta.url).pathname;
+
+const prompts = ["program"] as const;
+
+export const Prompts = Object.fromEntries(
+  await Promise.all(
+    prompts.map(async (prompt) => [
+      prompt,
+      await fs.readFile(path.join(__dirname, `${prompt}.md`), "utf-8"),
+    ]),
+  ),
+) as {
+  [key in (typeof prompts)[number]]: string;
+};

package/src/alchemize.ts

@@ -1,9 +1,8 @@
 import { evaluate } from "./apply";
 import { destroy } from "./destroy";
-import { defaultStage, nodes, providers, stateStore } from "./global";
-import type { StateStore } from "./state";
-
-let finalized = false;
+import { defaultStage, defaultStateStore, providers } from "./global";
+import { type Scope, getScope } from "./scope";
+import type { StateStoreType } from "./state";
 
 export interface AlchemizeOptions {
   /**
@@ -27,7 +26,17 @@ export interface AlchemizeOptions {
   /**
    * A custom state store to use instead of the default file system store.
    */
-  stateStore?: StateStore;
+  stateStore?: StateStoreType;
+  /**
+   * A custom scope to use instead of the default root scope.
+   */
+  scope?: Scope;
+  /**
+   * If true, will not print any Create/Update/Delete messages.
+   *
+   * @default false
+   */
+  quiet?: boolean;
 }
 
 /**
@@ -36,20 +45,21 @@ export interface AlchemizeOptions {
  * By default, this will be called when
  */
 export async function alchemize(options?: AlchemizeOptions) {
-  if (finalized) {
-    return;
-  }
-  finalized = true;
-
+  const scope = options?.scope ?? getScope();
+  const nodes = scope.nodes;
   const stage = options?.stage ?? defaultStage;
-  const state = options?.stateStore ?? stateStore;
+  const stateStore = new (options?.stateStore ?? defaultStateStore)(
+    scope.getScopePath(stage),
+  );
 
-  await state.init?.();
+  if (nodes.size > 0) {
+    await stateStore.init?.();
+  }
 
   // Track in-progress deletions to avoid duplicate work
   const deletionPromises = new Map<string, Promise<void>>();
 
-  const priorStates = await state.all(stage);
+  const priorStates = await stateStore.all();
   const priorIDs = Object.keys(priorStates);
 
   const mode = options?.mode ?? "up";
@@ -61,6 +71,9 @@ export async function alchemize(options?: AlchemizeOptions) {
         .map((node) =>
           evaluate(node.resource, {
             stage,
+            scope,
+            stateStore,
+            quiet: options?.quiet,
           }),
         ),
     );
@@ -78,6 +91,10 @@ export async function alchemize(options?: AlchemizeOptions) {
 
   const orphanIDs = Array.from(priorIDs).filter((id) => !aliveIDs.has(id));
 
+  console.log("orphanIDs", orphanIDs);
+  console.log("priorIDs", priorIDs);
+  console.log("aliveIDs", aliveIDs);
+
   const orphanStates = Object.fromEntries(
     orphanIDs.map((id) => [id, priorStates[id]] as const),
   );
@@ -89,6 +106,10 @@ export async function alchemize(options?: AlchemizeOptions) {
       (orphanGraph[dep] ??= new Set()).add(orphanID);
     }
   }
+  if (Object.keys(orphanGraph).length > 0) {
+    console.log(stateStore.path);
+    console.log(orphanGraph);
+  }
 
   // Start deletion from each orphan that has no dependents (nothing depends on it)
   await Promise.all(
@@ -97,6 +118,11 @@ export async function alchemize(options?: AlchemizeOptions) {
       .map((id) => deleteOrphan(id)),
   );
 
+  const remaining = await stateStore.count();
+  if (remaining === 0) {
+    await stateStore.deinit?.();
+  }
+
   // Recursively delete orphans in dependency order
   async function deleteOrphan(orphanID: string): Promise<void> {
     // Return existing deletion promise if this orphan is already being deleted
@@ -125,7 +151,12 @@ export async function alchemize(options?: AlchemizeOptions) {
           `No provider found for ${providerType}. Did you forget to import it?`,
         );
       }
-      await destroy(stage, orphanID, orphanState, provider);
+      if (!options?.quiet) {
+        console.log("Destroy:", orphanID);
+      }
+      await destroy(stage, scope, orphanID, orphanState, provider, stateStore, {
+        quiet: options?.quiet,
+      });
 
       // After this resource is deleted, we can delete its dependencies if they're orphans
       if (orphanState.deps.length > 0) {