alchemy 0.1.1 → 0.1.3

package/src/alchemize.ts

@@ -1,9 +1,8 @@
 import { evaluate } from "./apply";
 import { destroy } from "./destroy";
-import { defaultStage, nodes, providers, stateStore } from "./global";
-import type { StateStore } from "./state";
-
-let finalized = false;
+import { defaultStage, defaultStateStore, providers } from "./global";
+import { type Scope, getScope } from "./scope";
+import type { StateStoreType } from "./state";
 
 export interface AlchemizeOptions {
   /**
@@ -27,7 +26,17 @@ export interface AlchemizeOptions {
   /**
    * A custom state store to use instead of the default file system store.
    */
-  stateStore?: StateStore;
+  stateStore?: StateStoreType;
+  /**
+   * A custom scope to use instead of the default root scope.
+   */
+  scope?: Scope;
+  /**
+   * If true, will not print any Create/Update/Delete messages.
+   *
+   * @default false
+   */
+  quiet?: boolean;
 }
 
 /**
@@ -36,20 +45,21 @@ export interface AlchemizeOptions {
  * By default, this will be called when
  */
 export async function alchemize(options?: AlchemizeOptions) {
-  if (finalized) {
-    return;
-  }
-  finalized = true;
-
+  const scope = options?.scope ?? getScope();
+  const nodes = scope.nodes;
   const stage = options?.stage ?? defaultStage;
-  const state = options?.stateStore ?? stateStore;
+  const stateStore = new (options?.stateStore ?? defaultStateStore)(
+    scope.getScopePath(stage),
+  );
 
-  await state.init?.();
+  if (nodes.size > 0) {
+    await stateStore.init?.();
+  }
 
   // Track in-progress deletions to avoid duplicate work
   const deletionPromises = new Map<string, Promise<void>>();
 
-  const priorStates = await state.all(stage);
+  const priorStates = await stateStore.all();
   const priorIDs = Object.keys(priorStates);
 
   const mode = options?.mode ?? "up";
@@ -61,6 +71,9 @@ export async function alchemize(options?: AlchemizeOptions) {
         .map((node) =>
           evaluate(node.resource, {
             stage,
+            scope,
+            stateStore,
+            quiet: options?.quiet,
           }),
         ),
     );
@@ -78,6 +91,10 @@ export async function alchemize(options?: AlchemizeOptions) {
 
   const orphanIDs = Array.from(priorIDs).filter((id) => !aliveIDs.has(id));
 
+  console.log("orphanIDs", orphanIDs);
+  console.log("priorIDs", priorIDs);
+  console.log("aliveIDs", aliveIDs);
+
   const orphanStates = Object.fromEntries(
     orphanIDs.map((id) => [id, priorStates[id]] as const),
   );
@@ -89,6 +106,10 @@ export async function alchemize(options?: AlchemizeOptions) {
       (orphanGraph[dep] ??= new Set()).add(orphanID);
     }
   }
+  if (Object.keys(orphanGraph).length > 0) {
+    console.log(stateStore.path);
+    console.log(orphanGraph);
+  }
 
   // Start deletion from each orphan that has no dependents (nothing depends on it)
   await Promise.all(
@@ -97,6 +118,11 @@ export async function alchemize(options?: AlchemizeOptions) {
       .map((id) => deleteOrphan(id)),
   );
 
+  const remaining = await stateStore.count();
+  if (remaining === 0) {
+    await stateStore.deinit?.();
+  }
+
   // Recursively delete orphans in dependency order
   async function deleteOrphan(orphanID: string): Promise<void> {
     // Return existing deletion promise if this orphan is already being deleted
@@ -125,7 +151,12 @@ export async function alchemize(options?: AlchemizeOptions) {
           `No provider found for ${providerType}. Did you forget to import it?`,
         );
       }
-      await destroy(stage, orphanID, orphanState, provider);
+      if (!options?.quiet) {
+        console.log("Destroy:", orphanID);
+      }
+      await destroy(stage, scope, orphanID, orphanState, provider, stateStore, {
+        quiet: options?.quiet,
+      });
 
       // After this resource is deleted, we can delete its dependencies if they're orphans
       if (orphanState.deps.length > 0) {

package/src/apply.ts

@@ -1,4 +1,4 @@
-import { defaultStage, stateStore } from "./global";
+import { defaultStage, defaultStateStore } from "./global";
 import { Output } from "./output";
 import {
   Input,
@@ -7,11 +7,14 @@ import {
   ResourceID,
   isResource,
 } from "./resource";
+import { type Scope as IScope, getScope } from "./scope";
 import type { StateStore } from "./state";
 
-interface ApplyOptions {
-  stage?: string;
-  stateStore?: StateStore;
+export interface ApplyOptions {
+  stage: string;
+  scope: IScope;
+  stateStore: StateStore;
+  quiet?: boolean;
 }
 
 /**
@@ -21,9 +24,16 @@ interface ApplyOptions {
  */
 export async function apply<T>(
   output: T | Output<T>,
-  options?: ApplyOptions,
+  options?: Partial<ApplyOptions>,
 ): Promise<T> {
-  return (await evaluate(output, options)).value;
+  const stage = options?.stage ?? defaultStage;
+  const scope = options?.scope ?? getScope();
+  const statePath = scope.getScopePath(stage);
+  const stateStore = options?.stateStore ?? new defaultStateStore(statePath);
+
+  return (
+    await evaluate(output, { stage, scope, stateStore, quiet: options?.quiet })
+  ).value;
 }
 
 class Evaluated<T> {
@@ -37,9 +47,10 @@ const cache = new WeakMap<Resource, Promise<Evaluated<any>>>();
 
 export async function evaluate<T>(
   output: T | Output<T>,
-  options: ApplyOptions = {},
+  options: ApplyOptions,
 ): Promise<Evaluated<T>> {
-  const state = options.stateStore ?? stateStore;
+  const stage = options.stage;
+  const stateStore = options.stateStore;
   if (isResource(output)) {
     const resource = output;
     const resourceID = resource[ResourceID];
@@ -63,14 +74,16 @@ export async function evaluate<T>(
     const inputs = evaluated.map((input) => input.value);
     try {
       const result: T = await resource[Provider].update(
-        options.stage ?? defaultStage,
+        stage,
         resource,
         deps,
         inputs as [],
-        state,
+        stateStore,
+        options,
       );
       resolve!(new Evaluated(result, [resourceID, ...deps]));
     } catch (error) {
+      console.error(error);
       reject!(error);
     }
     return promise;
@@ -95,6 +108,8 @@ export async function evaluate<T>(
       evaluatedItems.map((e) => e.value) as unknown as T,
       evaluatedItems.flatMap((e) => e.deps),
     );
+  } else if (output instanceof Date) {
+    return new Evaluated(output as T);
   } else if (output && typeof output === "object") {
     const entries = Object.entries(output);
     const evaluatedEntries = await Promise.all(

package/src/{components/aws → aws}/bucket.ts

@@ -10,8 +10,8 @@ import {
   PutBucketTaggingCommand,
   S3Client,
 } from "@aws-sdk/client-s3";
-import { ignore } from "../../error";
-import { Resource } from "../../resource";
+import { ignore } from "../error";
+import { Resource } from "../resource";
 
 export interface BucketProps {
   bucketName: string;

package/src/{components/aws → aws}/function.ts

@@ -12,8 +12,8 @@ import {
 } from "@aws-sdk/client-lambda";
 import fs from "node:fs";
 import path from "node:path";
-import { ignore } from "../../error";
-import { Resource, type Context } from "../../resource";
+import { ignore } from "../error";
+import { type Context, Resource } from "../resource";
 
 async function resolveRegion(client: LambdaClient): Promise<string> {
   const region = client.config.region;

package/src/{components/aws → aws}/policy.ts

@@ -11,8 +11,8 @@ import {
   ListPolicyVersionsCommand,
   NoSuchEntityException,
 } from "@aws-sdk/client-iam";
-import { ignore } from "../../error";
-import { Resource } from "../../resource";
+import { ignore } from "../error";
+import { Resource } from "../resource";
 
 // Type-safe policy document types
 export type Effect = "Allow" | "Deny";

package/src/{components/aws → aws}/queue.ts

@@ -5,7 +5,7 @@ import {
   GetQueueUrlCommand,
   SQSClient,
 } from "@aws-sdk/client-sqs";
-import { Resource } from "../../resource";
+import { Resource } from "../resource";
 
 export interface QueueProps {
   queueName: string;

package/src/{components/aws → aws}/role.ts

@@ -12,8 +12,8 @@ import {
   UpdateAssumeRolePolicyCommand,
   UpdateRoleCommand,
 } from "@aws-sdk/client-iam";
-import { ignore } from "../../error";
-import { type Context, Resource } from "../../resource";
+import { ignore } from "../error";
+import { type Context, Resource } from "../resource";
 import type { PolicyDocument } from "./policy";
 
 export interface RoleProps {

package/src/{components/aws → aws}/table.ts

@@ -7,8 +7,8 @@ import {
   ResourceInUseException,
   ResourceNotFoundException,
 } from "@aws-sdk/client-dynamodb";
-import { ignore } from "../../error";
-import { type Context, Resource } from "../../resource";
+import { ignore } from "../error";
+import { type Context, Resource } from "../resource";
 
 export interface TableProps {
   tableName: string;

package/src/destroy.ts

@@ -1,10 +1,14 @@
-import { defaultStage, stateStore } from "./global";
+import { defaultStage, defaultStateStore, rootScope } from "./global";
 import type { Output } from "./output";
 import { Provider, ResourceID, isResource } from "./resource";
-import type { State } from "./state";
+import type { Scope } from "./scope";
+import type { State, StateStore } from "./state";
 
-interface DestroyOptions {
+export interface DestroyOptions {
   stage?: string;
+  stateStore?: StateStore;
+  scope?: Scope;
+  quiet?: boolean;
 }
 
 /**
@@ -20,44 +24,64 @@ export async function destroy<T>(
  */
 export async function destroy<T>(
   stage: string,
+  scope: Scope,
   resourceID: ResourceID,
   resourceState: State,
   resourceProvider: Provider,
+  stateStore: StateStore,
+  options: DestroyOptions,
 ): Promise<void>;
 
 export async function destroy<T>(
-  ...args: [Output<T>, DestroyOptions?] | [string, ResourceID, State, Provider]
+  ...args:
+    | [Output<T>, DestroyOptions?]
+    | [string, Scope, ResourceID, State, Provider, StateStore, DestroyOptions]
 ): Promise<void> {
   let resourceID: ResourceID;
   let resourceState: State;
   let resourceProvider: Provider;
   let stage: string;
-
-  if (args.length === 4) {
+  let stateStore: StateStore;
+  let scope: Scope | undefined = undefined;
+  let options: DestroyOptions | undefined = undefined;
+  if (args.length === 7) {
     stage = args[0];
-    resourceID = args[1];
-    resourceState = args[2];
-    resourceProvider = args[3];
+    scope = args[1];
+    resourceID = args[2];
+    resourceState = args[3];
+    resourceProvider = args[4];
+    stateStore = args[5];
+    options = args[6];
   } else if (isResource(args[0])) {
     const resource = args[0];
-    stage = args[1]?.stage ?? defaultStage;
+    // stage = args[1]?.stage ?? defaultStage;
     resourceID = resource[ResourceID];
+    stage = args[1]?.stage ?? defaultStage;
+    scope = args[1]?.scope ?? rootScope;
+    const statePath = scope.getScopePath(stage);
+    stateStore = args[1]?.stateStore ?? new defaultStateStore(statePath);
     // First destroy all dependencies
-    const _resourceState = await stateStore.get(stage, resourceID);
+    const _resourceState = await stateStore.get(resourceID);
     if (_resourceState === undefined) {
       // we have no record of this resource, we must assume it's already deleted
       return;
     }
     resourceState = _resourceState;
     resourceProvider = resource[Provider];
+    options = args[1];
   } else {
+    console.log(args[0]);
     throw new Error("Not implemented: must handle destroy a Output chain");
   }
   await resourceProvider.delete(
     stage,
+    scope,
     resourceID,
     resourceState,
     resourceState.inputs as [],
+    options ?? {
+      quiet: false,
+    },
   );
-  await stateStore.delete(stage, resourceID);
+  await stateStore.delete(resourceID);
 }

package/src/{components/esbuild.ts → esbuild.ts}

@@ -2,7 +2,7 @@ import * as esbuild from "esbuild";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { Resource } from "../resource";
+import { Resource } from "./resource";
 
 export interface BundleProps {
   /**

package/src/{components/fs.ts → fs.ts}

@@ -1,7 +1,8 @@
 import fs from "node:fs";
+import { unlink } from "node:fs/promises";
 import path from "node:path";
-import { ignore } from "../error";
-import { Resource } from "../resource";
+import { ignore } from "./error";
+import { Resource } from "./resource";
 
 export class File extends Resource(
   "fs::File",
@@ -18,28 +19,29 @@ export class File extends Resource(
 
 export class Folder extends Resource(
   "fs::Folder",
-  async (ctx, dirPath: string) => {
+  async (ctx, dirPath: string): Promise<{ path: string }> => {
     if (ctx.event === "delete") {
-      if (await ctx.get("isOwned")) {
-        // only delete the folder if we own it (we created it)
-        await ignore("ENOENT", () => fs.promises.rmdir(dirPath));
-      }
+      // we just do a best effort attempt
+      await ignore(["ENOENT", "ENOTEMPTY"], () => fs.promises.rmdir(dirPath));
     } else {
-      try {
-        await fs.promises.mkdir(dirPath, { recursive: true });
-      } catch (error: any) {
-        if (error.code === "EEXIST") {
-          // record that we own the directory
-          await ctx.set("isOwned", true);
-        } else {
-          throw error;
-        }
-      }
+      await ignore("EEXIST", () =>
+        fs.promises.mkdir(dirPath, { recursive: true }),
+      );
     }
-    return dirPath;
+    return { path: dirPath };
   },
 ) {
   public file(filePath: string, content: string) {
     return new File(path.basename(filePath), filePath, content);
   }
 }
+
+export async function rm(filePath: string) {
+  try {
+    await unlink(filePath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw error;
+    }
+  }
+}

package/src/global.ts

@@ -1,11 +1,12 @@
 import fs from "node:fs";
 import path from "node:path";
-import type { Provider, Resource, ResourceID, ResourceType } from "./resource";
-import { defaultStore, type StateStore } from "./state";
+import type { Provider, ResourceType } from "./resource";
+import { Scope } from "./scope";
+import { FileSystemStateStore } from "./state";
 
 export interface Config {
   defaultStage?: string;
-  stateStore?: StateStore;
+  stateStore?: new (dir: string) => FileSystemStateStore;
 }
 
 const alchemy = path.resolve(process.cwd(), "alchemy.ts");
@@ -43,20 +44,14 @@ const _stage = process.env.ALCHEMY_STAGE ?? process.env.USER ?? "dev";
 
 export const config: Config = _config ?? {
   defaultStage: _stage,
-  stateStore: defaultStore,
+  stateStore: FileSystemStateStore,
 };
 
 export const defaultStage = config.defaultStage ?? _stage;
 
-export const stateStore = config.stateStore ?? defaultStore;
+export const defaultStateStore = config.stateStore ?? FileSystemStateStore;
 
-export const nodes = new Map<
-  ResourceID,
-  {
-    provider: Provider<any, any[], any>;
-    resource: Resource<any, any>;
-  }
->();
+export const rootScope = new Scope(null);
 
 export const providers = new Map<
   ResourceType,
@@ -68,5 +63,3 @@ export const deletions: {
   data: Record<string, any>;
   inputs: any[];
 }[] = [];
-
-await stateStore.init?.();

package/src/index.ts

@@ -1,6 +1,15 @@
+export * from "./$.ts";
 export * from "./alchemize";
+export * from "./error";
 export * from "./global";
+export * from "./input";
 export * from "./output";
-export * from "./resource";
+export {
+  Resource,
+  type Context,
+  type CreateContext,
+  type DeleteContext,
+  type UpdateContext,
+} from "./resource";
 
-export type { StateStore } from "./state";
+export type { FileSystemStateStore as StateStore } from "./state";

package/src/input.ts

@@ -4,7 +4,7 @@ export type Inputs<P extends any[]> = P extends [infer First, ...infer Rest]
   ? [Input<First>, ...Inputs<Rest>]
   : [];
 
-export type Input<T> =
+export type Input<T = any> =
   | T
   | Output<T>
   | (T extends any[] ? Inputs<T> : never)

package/src/markdown/design.ts

@@ -0,0 +1,125 @@
+import { tool } from "ai";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import {
+  type FileContext,
+  type ModelId,
+  dependenciesAsMessages,
+  generateText,
+  resolveModel,
+} from "../agent";
+import { Prompts } from "../agent/prompts";
+import { rm } from "../fs";
+import { extractMarkdown } from "../markdown/extract";
+import { type Context, Resource } from "../resource";
+
+export interface DesignInput {
+  prompt: string;
+  path: string;
+  deps?: any[];
+  model?: ModelId;
+  dependencies: FileContext[];
+}
+
+export interface DesignOutput {
+  path: string;
+  content: string;
+}
+
+export class Design extends Resource(
+  "design",
+  async (ctx: Context<DesignOutput>, props: DesignInput) => {
+    if (ctx.event === "delete") {
+      await rm(props.path);
+      return;
+    }
+    const model = await resolveModel(props.model ?? "gpt-4o");
+
+    const importFile = tool({
+      description: "Imports a relevant file into context",
+      parameters: z.object({
+        file: z.string(),
+      }),
+      execute: async ({ file }) => fs.readFile(file, "utf-8"),
+    });
+
+    let existingContent: string | undefined = undefined;
+    if (ctx.event === "update") {
+      if (props.path) {
+        try {
+          existingContent = await fs.readFile(props.path, "utf-8");
+        } catch {
+          // File doesn't exist yet, continue with empty content
+        }
+      } else {
+        existingContent = ctx.output.content;
+      }
+    }
+
+    console.log(
+      ctx.event === "create" ? "Designing" : "Revising design",
+      ":",
+      props.path,
+    );
+
+    const result = await generateText({
+      model,
+      // tools: {
+      //   import: importFile,
+      // },
+      providerOptions:
+        props.model === "o3-mini"
+          ? {
+              openai: {
+                reasoningEffort: "high",
+              },
+            }
+          : undefined,
+      messages: [
+        {
+          role: "system",
+          content: Prompts.program,
+        },
+        ...dependenciesAsMessages(props.dependencies),
+        ...(existingContent
+          ? [
+              {
+                role: "user" as const,
+                content: `Here is the prior version of ${props.path}:\n${existingContent}`,
+              },
+              {
+                role: "assistant" as const,
+                content:
+                  "Thanks, I'll make sure to consider this when producing the next revision.",
+              },
+            ]
+          : []),
+        {
+          role: "user",
+          content: props.prompt,
+        },
+        {
+          role: "user",
+          content: [
+            `Make sure to list each of the files that need to be implemented and`,
+            "and to follow best practices in term of separation of concerns.",
+            "Ideally, each component has its own dedicated file.",
+            "Each file should be referenced using markdown links, e.g. [file.ts](./file.ts).",
+          ].join(" "),
+        },
+      ],
+    });
+
+    const content = extractMarkdown(result.text);
+
+    await fs.mkdir(path.dirname(props.path), { recursive: true });
+
+    await fs.writeFile(props.path, content);
+
+    return {
+      path: props.path,
+      content,
+    };
+  },
+) {}

package/src/markdown/extract.ts

@@ -0,0 +1,9 @@
+/**
+ * Extracts markdown content from a code block or returns the full content
+ * @param text Response text that may contain markdown code blocks
+ * @returns The markdown content from code block if found, otherwise returns the full text
+ */
+export function extractMarkdown(text: string): string {
+  const matches = text.match(/```md\n([\s\S]*?)```/);
+  return matches?.[1]?.trim() ?? text.trim();
+}

package/src/markdown/index.ts

@@ -0,0 +1,3 @@
+export * from "./design";
+export * from "./extract";
+export * from "./requirements";