alchemy 2.0.0-beta.74 → 2.0.0-beta.76

package/src/Docker/BuildHash.ts

@@ -0,0 +1,308 @@
+/**
+ * INTERNAL — deterministic identity for a Docker build: context + Dockerfile
+ * + platform + build args, with Docker's own semantics (`.dockerignore`
+ * pattern matching, symlink targets, permission bits, empty directories).
+ * Cloud-agnostic — consumed by `AWS.ECR.Image` and `AWS.Lambda.Function`
+ * image packaging; NOT exported from the Docker barrel.
+ */
+
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Result from "effect/Result";
+import * as Stream from "effect/Stream";
+import * as crypto from "node:crypto";
+
+export interface DockerBuildSource {
+  context: string;
+  dockerfile: string;
+  platform: string;
+  buildArgs?: Record<string, string>;
+}
+
+export type DockerBuildHashMode = "all" | "effective";
+
+interface DockerIgnoreRule {
+  ignored: boolean;
+  expression: RegExp;
+}
+
+interface DockerIgnore {
+  content: string;
+  path?: string;
+  rules: ReadonlyArray<DockerIgnoreRule>;
+}
+
+const normalizeRelativePath = (value: string) =>
+  value.replaceAll("\\", "/").replace(/^\.\/+/, "");
+
+const escapeRegExp = (value: string) =>
+  value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+
+/**
+ * Compile Docker's ordered ignore-pattern form into a path matcher.
+ *
+ * Matching is rooted at the context. Descendants of a matched directory are
+ * handled by checking each path's parents in {@link isDockerIgnored}.
+ */
+const compileDockerIgnoreRule = (raw: string): DockerIgnoreRule | undefined => {
+  if (raw.startsWith("#")) {
+    return undefined;
+  }
+
+  let pattern = raw.trim();
+  if (pattern.length === 0 || pattern === ".") {
+    return undefined;
+  }
+
+  let ignored = true;
+  if (pattern.startsWith("\\!") || pattern.startsWith("\\#")) {
+    pattern = pattern.slice(1);
+  } else if (pattern.startsWith("!")) {
+    ignored = false;
+    pattern = pattern.slice(1).trim();
+  }
+
+  pattern = pattern
+    .replace(/^\.\/+/, "")
+    .replace(/^\/+/, "")
+    .replace(/\/+$/, "");
+  if (pattern.length === 0 || pattern === ".") {
+    return undefined;
+  }
+
+  let body = "";
+  for (let index = 0; index < pattern.length; index++) {
+    const char = pattern[index];
+    if (char === "\\" && pattern[index + 1] !== undefined) {
+      body += escapeRegExp(pattern[++index]);
+      continue;
+    }
+    if (char === "*") {
+      if (pattern[index + 1] === "*") {
+        while (pattern[index + 1] === "*") {
+          index++;
+        }
+        if (pattern[index + 1] === "/") {
+          index++;
+          body += "(?:.*/)?";
+        } else {
+          body += ".*";
+        }
+      } else {
+        body += "[^/]*";
+      }
+      continue;
+    }
+    if (char === "?") {
+      body += "[^/]";
+      continue;
+    }
+    if (char === "[") {
+      const end = pattern.indexOf("]", index + 1);
+      if (end !== -1) {
+        const content = pattern.slice(index + 1, end);
+        const negated = content.startsWith("!") || content.startsWith("^");
+        const members = negated ? content.slice(1) : content;
+        body += `[${negated ? "^" : ""}${members.replaceAll("\\", "\\\\")}]`;
+        index = end;
+        continue;
+      }
+    }
+    body += escapeRegExp(char);
+  }
+
+  return {
+    ignored,
+    expression: new RegExp(`^${body}$`),
+  };
+};
+
+const isDockerIgnored = (
+  relativePath: string,
+  rules: ReadonlyArray<DockerIgnoreRule>,
+) => {
+  const segments = normalizeRelativePath(relativePath).split("/");
+  const candidates = segments.map((_, index) =>
+    segments.slice(0, index + 1).join("/"),
+  );
+  let ignored = false;
+  for (const rule of rules) {
+    if (candidates.some((candidate) => rule.expression.test(candidate))) {
+      ignored = rule.ignored;
+    }
+  }
+  return ignored;
+};
+
+/**
+ * Resolve and validate a Docker build context and Dockerfile.
+ *
+ * Dockerfile paths are relative to the build context unless absolute.
+ */
+export const resolveDockerBuildPaths = Effect.fn(function* (
+  source: Pick<DockerBuildSource, "context" | "dockerfile">,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const context = path.resolve(source.context);
+  const dockerfile = path.isAbsolute(source.dockerfile)
+    ? source.dockerfile
+    : path.resolve(context, source.dockerfile);
+
+  if (!(yield* fs.exists(context))) {
+    return yield* Effect.fail(
+      new Error(`Docker build context does not exist: ${context}`),
+    );
+  }
+  if (!(yield* fs.exists(dockerfile))) {
+    return yield* Effect.fail(
+      new Error(`Dockerfile does not exist: ${dockerfile}`),
+    );
+  }
+
+  return { context, dockerfile };
+});
+
+const resolveDockerIgnore = Effect.fn(function* ({
+  context,
+  dockerfile,
+}: {
+  context: string;
+  dockerfile: string;
+}) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const dockerfileIgnore = `${dockerfile}.dockerignore`;
+  const contextIgnore = path.join(context, ".dockerignore");
+  const ignoreFile = (yield* fs.exists(dockerfileIgnore))
+    ? dockerfileIgnore
+    : (yield* fs.exists(contextIgnore))
+      ? contextIgnore
+      : undefined;
+
+  if (ignoreFile === undefined) {
+    return undefined;
+  }
+
+  const content = yield* fs.readFileString(ignoreFile);
+  const relativePath = normalizeRelativePath(
+    path.relative(context, ignoreFile),
+  );
+  return {
+    content,
+    path:
+      relativePath === ".." ||
+      relativePath.startsWith("../") ||
+      path.isAbsolute(relativePath)
+        ? undefined
+        : relativePath,
+    rules: content.split(/\r?\n/).flatMap((line) => {
+      const rule = compileDockerIgnoreRule(line);
+      return rule === undefined ? [] : [rule];
+    }),
+  } satisfies DockerIgnore;
+});
+
+/**
+ * Hash a Docker build context together with the Dockerfile, platform, and
+ * build arguments. In `effective` mode, the selected `.dockerignore` is
+ * applied exactly once before files are hashed. Absolute paths do not
+ * participate.
+ */
+export const hashDockerBuildInputs = Effect.fn(function* (
+  source: DockerBuildSource,
+  mode: DockerBuildHashMode,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const { context, dockerfile } = yield* resolveDockerBuildPaths(source);
+  const dockerignore =
+    mode === "effective"
+      ? yield* resolveDockerIgnore({ context, dockerfile })
+      : undefined;
+  const hasher = yield* Effect.sync(() => crypto.createHash("sha256"));
+
+  yield* Effect.sync(() =>
+    hasher.update(
+      JSON.stringify({
+        platform: source.platform,
+        buildArgs: Object.entries(source.buildArgs ?? {}).sort(([a], [b]) =>
+          a < b ? -1 : a > b ? 1 : 0,
+        ),
+        dockerignore:
+          dockerignore === undefined
+            ? undefined
+            : {
+                content: dockerignore.content,
+                path: dockerignore.path,
+              },
+      }),
+    ),
+  );
+
+  const dockerfileContent = yield* fs.readFile(dockerfile);
+  yield* Effect.sync(() => {
+    hasher.update("Dockerfile\0");
+    hasher.update(dockerfileContent);
+  });
+
+  const entries = yield* fs.readDirectory(context, { recursive: true });
+  for (const entry of entries.sort()) {
+    const normalizedEntry = normalizeRelativePath(entry);
+    if (
+      dockerignore !== undefined &&
+      (normalizedEntry === dockerignore.path ||
+        isDockerIgnored(normalizedEntry, dockerignore.rules))
+    ) {
+      continue;
+    }
+
+    const fullPath = path.join(context, entry);
+    const hashedEntry = mode === "effective" ? normalizedEntry : entry;
+    // FileSystem.stat follows symbolic links, so probe the link itself first.
+    // Docker preserves the link in the build context; hashing the target file
+    // would miss retargets between files with identical contents and metadata.
+    const link = yield* Effect.result(fs.readLink(fullPath));
+    if (Result.isSuccess(link)) {
+      yield* Effect.sync(() =>
+        hasher.update(
+          `${JSON.stringify({
+            path: hashedEntry,
+            type: "SymbolicLink",
+            target: link.success,
+          })}\0`,
+        ),
+      );
+      continue;
+    }
+
+    const info = yield* fs.stat(fullPath);
+
+    // Docker COPY preserves entry types and permission bits, including empty
+    // directories. Include that metadata before any file bytes so changes such
+    // as making a Lambda bootstrap executable invalidate the image tag.
+    yield* Effect.sync(() =>
+      hasher.update(
+        `${JSON.stringify({
+          path: hashedEntry,
+          type: info.type,
+          mode: info.mode & 0o7777,
+          size: info.type === "File" ? String(info.size) : undefined,
+        })}\0`,
+      ),
+    );
+
+    if (info.type === "File") {
+      yield* fs.stream(fullPath).pipe(
+        Stream.runForEach((chunk) =>
+          Effect.sync(() => {
+            hasher.update(chunk);
+          }),
+        ),
+      );
+    }
+  }
+
+  return (yield* Effect.sync(() => hasher.digest("hex"))).slice(0, 32);
+});

package/src/Docker/Container.ts

@@ -45,6 +45,18 @@ export interface ContainerProps {
   stopTimeout?: Duration.Input;
   /** Networks to connect after create. */
   networks?: Container.NetworkMapping[];
+  /**
+   * Extra `/etc/hosts` entries, each `hostname:address`. Docker's
+   * `host-gateway` alias resolves to the host machine, so
+   * `"host.docker.internal:host-gateway"` reaches services listening on the
+   * developer's machine from inside the container.
+   *
+   * On Linux `host-gateway` is the bridge gateway address, so those packets
+   * traverse the host's `INPUT` chain — under a default-deny firewall the
+   * name resolves and the connection then times out. See the Host Access
+   * examples.
+   */
+  extraHosts?: string[];
   /** Remove the container when it exits. @default false */
   removeOnExit?: boolean;
   /** Start the container after creation/reconciliation. @default false */
@@ -174,6 +186,55 @@ export interface Container extends Resource<
  * const runtime = yield* Docker.inspectContainer(postgresName);
  * ```
  *
+ * ### Host Access
+ * `extraHosts` writes lines into the container's `/etc/hosts`; it changes name
+ * resolution and nothing else. Docker's `host-gateway` alias resolves to the
+ * host machine, which is how a container reaches a service on the developer's
+ * loopback.
+ *
+ * On Linux `host-gateway` is the Docker bridge gateway (typically
+ * `172.17.0.1`), so a container's packets to it arrive on the host's `INPUT`
+ * chain. Under a default-deny firewall — ufw ships
+ * `DEFAULT_INPUT_POLICY="DROP"` — the hostname resolves correctly and the
+ * connection then times out, which reads like an application bug rather than a
+ * firewall one. Allow the bridge subnet to fix it:
+ * `sudo ufw allow from 172.16.0.0/12`.
+ *
+ * **Example:** Reach a service on the developer's machine
+ * ```typescript
+ * const api = yield* Docker.Container("api", {
+ *   image: "ghcr.io/acme/api:latest",
+ *   // `host-gateway` resolves to the host machine, so a database listening
+ *   // on the developer's loopback is reachable from inside the container.
+ *   extraHosts: ["host.docker.internal:host-gateway"],
+ *   environment: {
+ *     DATABASE_URL: "postgres://postgres@host.docker.internal:5432/app",
+ *   },
+ *   start: true,
+ * });
+ * ```
+ *
+ * **Example:** Pin a hostname to a fixed address
+ * ```typescript
+ * const api = yield* Docker.Container("api", {
+ *   image: "ghcr.io/acme/api:latest",
+ *   // Any `hostname:address` pair — host access is just the common case.
+ *   extraHosts: ["payments.internal:10.1.2.3"],
+ *   start: true,
+ * });
+ * ```
+ *
+ * **Example:** Publish on any free host port
+ * ```typescript
+ * const api = yield* Docker.Container("api", {
+ *   image: "ghcr.io/acme/api:latest",
+ *   // `external: 0` lets Docker choose; the assigned port is reported back.
+ *   ports: [{ external: 0, internal: 3000 }],
+ *   start: true,
+ * });
+ * const hostPort = api.ports["3000/tcp"];
+ * ```
+ *
  * ### Traefik
  * **Example:** Route a container through Traefik
  * ```typescript
@@ -235,11 +296,11 @@ export const ContainerProvider = () =>
       const reconcileNetworks = Effect.fn(function* (
         live: Docker.Container,
         news: ContainerProps,
+        olds: ContainerProps | undefined,
       ) {
         const context = dockerContextName(news.context);
         const connect = new Map<string, Container.NetworkMapping>();
         const disconnect = new Set<string>();
-        const noop = new Set<string>();
         for (const network of news.networks ?? []) {
           const entry = live.NetworkSettings.Networks?.[network.name];
           if (!entry) {
@@ -249,13 +310,20 @@ export const ContainerProvider = () =>
           ) {
             connect.set(network.name, network);
             disconnect.add(network.name);
-          } else {
-            noop.add(network.name);
           }
         }
-        for (const key of Object.keys(live.NetworkSettings.Networks ?? {})) {
-          if (!noop.has(key)) {
-            disconnect.add(key);
+        // Only networks alchemy itself attached are alchemy's to detach, and
+        // `olds.networks` is the sole record of which those are — the live
+        // container cannot say who connected a network. Sweeping every live
+        // network instead tore off the default `bridge` and anything a user,
+        // compose file, or another tool had attached out of band (#1386).
+        const desired = new Set((news.networks ?? []).map((n) => n.name));
+        for (const network of olds?.networks ?? []) {
+          if (
+            !desired.has(network.name) &&
+            live.NetworkSettings.Networks?.[network.name]
+          ) {
+            disconnect.add(network.name);
           }
         }
         yield* Effect.forEach(
@@ -333,7 +401,7 @@ export const ContainerProvider = () =>
             return { action: "update" as const };
           }
         }),
-        reconcile: Effect.fn(function* ({ id, instanceId, news }) {
+        reconcile: Effect.fn(function* ({ id, instanceId, news, olds }) {
           const context = dockerContextName(news.context);
           const args = yield* makeCreateArgs(id, news, instanceId);
           const live = yield* docker.container
@@ -347,7 +415,7 @@ export const ContainerProvider = () =>
             );
 
           if (live) {
-            yield* reconcileNetworks(live, news);
+            yield* reconcileNetworks(live, news, olds);
             if (news.start && live.State.Status !== "running") {
               yield* docker.container.start(live.Id, context);
             } else if (!news.start && live.State.Status === "running") {
@@ -419,10 +487,17 @@ const makeCreateArgs = (id: string, news: ContainerProps, instanceId: string) =>
         volume: news.volumes?.map(
           (v) => `${v.hostPath}:${v.containerPath}${v.readOnly ? ":ro" : ""}`,
         ),
-        p: news.ports?.map(
-          (port) =>
-            `${port.external}:${port.internal}/${port.protocol ?? "tcp"}`,
-        ),
+        p: news.ports?.map((port) => {
+          const target = `${port.internal}/${port.protocol ?? "tcp"}`;
+          // `external: 0` means "any free host port". Docker spells that as a
+          // bare container port (`-p 80/tcp`); `-p 0:80/tcp` instead asks for
+          // host port 0 literally, which the daemon accepts and then reports
+          // back as 0.
+          return isRandomHostPort(port.external)
+            ? target
+            : `${port.external}:${target}`;
+        }),
+        "add-host": news.extraHosts,
         restart: news.restart ?? "no",
         label: news.labels,
         "stop-timeout": toSeconds(news.stopTimeout)?.toString(),
@@ -463,17 +538,51 @@ const toContainerAttributes = (
   status: info.State.Status,
   createdAt: Date.parse(info.Created) || Date.now(),
   imageRef,
-  ports: Object.fromEntries(
-    Object.entries({
-      ...info.NetworkSettings.Ports,
-      ...info.HostConfig.PortBindings,
-    }).flatMap(([internal, bindings]) => {
-      if (!bindings?.[0]?.HostPort) return [];
-      return [[internal, Number.parseInt(bindings[0].HostPort, 10)]];
-    }),
-  ),
+  ports: toPortAttributes(info),
 });
 
+/** First binding that carries a real (non-zero) host port. */
+const boundHostPort = (
+  bindings: ReadonlyArray<{ HostPort?: string }> | null | undefined,
+): number | undefined => {
+  for (const binding of bindings ?? []) {
+    if (!binding.HostPort) continue;
+    const port = Number.parseInt(binding.HostPort, 10);
+    if (Number.isInteger(port) && port > 0) return port;
+  }
+  return undefined;
+};
+
+/**
+ * `HostConfig.PortBindings` is what was *requested*, `NetworkSettings.Ports`
+ * what Docker actually *assigned* — so the assignment wins wherever both
+ * exist. A container published with `external: 0` (or any random-publish
+ * mapping) has no requested host port at all, and reading the request over
+ * the assignment reported 0 instead of the port the container is reachable
+ * on. The request is still the fallback: a created-but-not-yet-started
+ * container has empty `NetworkSettings.Ports`.
+ */
+const toPortAttributes = (info: Docker.Container): Record<string, number> => {
+  const ports: Record<string, number> = {};
+  for (const [internal, bindings] of Object.entries(
+    info.HostConfig.PortBindings ?? {},
+  )) {
+    const port = boundHostPort(bindings);
+    if (port !== undefined) ports[internal] = port;
+  }
+  for (const [internal, bindings] of Object.entries(
+    info.NetworkSettings.Ports ?? {},
+  )) {
+    const port = boundHostPort(bindings);
+    if (port !== undefined) ports[internal] = port;
+  }
+  return ports;
+};
+
+/** `external: 0` / `"0"` asks Docker to pick any free host port. */
+const isRandomHostPort = (external: number | string): boolean =>
+  Number.parseInt(String(external), 10) === 0;
+
 const normalizeEnvironment = (
   environment: Record<string, string | Redacted.Redacted<string>> | undefined,
 ): Record<string, string> =>

package/src/Docker/Docker.ts

@@ -51,6 +51,8 @@ export class Docker extends Context.Service<
         "health-start-interval": string | undefined;
         "stop-timeout": string | undefined;
         p: Array<string> | undefined;
+        /** `--add-host` entries, each `hostname:address`. */
+        "add-host"?: Array<string> | undefined;
         command: Array<string> | undefined;
         label?: Record<string, string>;
         context?: string;
@@ -397,6 +399,7 @@ export declare namespace Docker {
         Array<{ HostIp: string; HostPort: string }> | null
       > | null;
       Binds: string[] | null;
+      ExtraHosts: string[] | null;
       RestartPolicy: {
         Name: string;
         MaximumRetryCount: number;

package/src/Docker/ServiceImage.ts

@@ -7,7 +7,6 @@ import {
   getStableContextDir,
   resolveMainPath,
 } from "../Bundle/TempRoot.ts";
-import { Self } from "../Self.ts";
 import { sha256Object } from "../Util/sha256.ts";
 import { Docker } from "./Docker.ts";
 
@@ -50,85 +49,18 @@ export interface ResolvedServiceImage {
 }
 
 /**
- * The standard bun bootstrap wrapped around a `Docker.Service` `main` entry:
- * resolves the bundled program's registered runners (`host.run` loops and the
- * served `fetch` handler) and runs them with a Bun HTTP server bound to
- * `PORT`. Mirrors the ECS bootstrap minus the AWS credential/region layers —
- * a Swarm task has no ambient cloud identity.
+ * The generated entry for `Docker.Service` containers: a shim importing only
+ * `alchemy/Runtime/Bootstrap/Docker` plus the user's `main` — see that
+ * module for why the entry never imports alchemy's own dependencies.
  */
 export const makeServiceBunBootstrap =
   (handler: string) =>
   (importPath: string): string =>
     `
-import { BunServices } from "@effect/platform-bun";
-import { BunHttpServer } from "alchemy/Http";
-import { Stack } from "alchemy/Stack";
-import { makeEntrypointLayer, reifyBoundConfigProvider } from "alchemy/Runtime";
-import * as Config from "effect/Config";
-import * as ConfigProvider from "effect/ConfigProvider";
-import * as Context from "effect/Context";
-import * as Effect from "effect/Effect";
-import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient";
-import * as Layer from "effect/Layer";
-import * as Logger from "effect/Logger";
-
+import { bootstrap } from "alchemy/Runtime/Bootstrap/Docker";
 import { ${handler} as entrypoint } from ${JSON.stringify(importPath)};
 
-// Normalize the entrypoint export: an inline-effect class default export is
-// an Effect resolving the platform instance, while the tagged form
-// (X.make(props, impl)) exports a Layer providing the Self tag. Both fold
-// into a Layer via makeEntrypointLayer (same pattern as the ECS bridge).
-const tag = Context.Service("${Self.key}");
-const layer = makeEntrypointLayer(tag, entrypoint);
-
-const platform = Layer.mergeAll(
-  BunServices.layer,
-  FetchHttpClient.layer,
-  Logger.layer([Logger.consolePretty()]),
-);
-
-const stack = Layer.effect(
-  Stack,
-  Effect.all([
-    Config.string("ALCHEMY_STACK_NAME"),
-    Config.string("ALCHEMY_STAGE")
-  ]).pipe(
-    Effect.map(([name, stage]) => ({
-      name,
-      stage,
-      bindings: {},
-      resources: {}
-    }))
-  )
-);
-
-// Resolve the bundled program (the runners registered via host.run / serve)
-// and run it with a Bun HTTP server bound to PORT, so the returned { fetch }
-// handler is actually served and host.run loops stay alive.
-const program = tag.pipe(
-  Effect.flatMap((service) => service.RuntimeContext.exports),
-  Effect.flatMap((exports) => exports.program),
-  Effect.provide(
-    layer.pipe(
-      Layer.provideMerge(stack),
-      Layer.provideMerge(BunHttpServer()),
-      Layer.provideMerge(platform),
-      Layer.provideMerge(
-        Layer.succeed(
-          ConfigProvider.ConfigProvider,
-          reifyBoundConfigProvider(ConfigProvider.fromEnv(), process.env)
-        )
-      ),
-    )
-  ),
-  Effect.scoped
-);
-
-console.log("Docker service bootstrap starting...");
-await Effect.runPromise(program).catch((err) => {
-  console.error("Docker service bootstrap failed:", err);
-  process.exit(1);
-});
+await bootstrap(entrypoint);
 `;
 
 /**
@@ -172,7 +104,7 @@ export const makeServiceImage = Effect.gen(function* () {
             ...((source.build?.input?.external as string[] | undefined) ?? []),
           ],
           resolve: {
-            conditionNames: ["bun", "import", "module", "default"],
+            conditionNames: [...Bundle.BUN_CONDITION_NAMES],
             ...source.build?.input?.resolve,
           },
           plugins: [source.build?.input?.plugins, plugins],