flisk-import 0.5.0 → 0.7.0

package/README.md

@@ -10,12 +10,13 @@ flisk import . --yes         # write service.json files and project setup
 
 ## What to expect
 
-- **Reads one file:** the root `docker-compose.yml`. Not `compose.yaml`, override files, nested Compose files, Dockerfiles on their own, `package.json`, or Procfiles.
+- **Reads one file:** the root `docker-compose.yml`, or `compose.yaml`, `compose.yml` or `docker-compose.yaml` when that is absent. Not override files, nested Compose files, Dockerfiles on their own, `package.json`, or Procfiles.
 - **Writes:** one bring-your-own `service.json` per Compose service, plus `.flisk/project.json` if the repo is not a Flisk project yet. It never edits the Compose file or overwrites a `service.json` without `--force`.
 - **Keeps:** the service names (`db` stays `db`, so it stays the hostname), the build context and Dockerfile (or the image), the container port as written (3000, 80, 5432), `depends_on`, and an HTTP health path. Databases and brokers are private, with no proxy route.
 - **Runtime keys:** named volumes and bind mounts inside the project (`volumes`), `cap_add`, `cap_drop`, `init`, `read_only`, `user`, `security_opt` and `shm_size` (`container`), extra published ports (`extraPorts`, on 127.0.0.1 only), `network_mode: service:<name>` (`sidecarOf`) and `restart`.
-- **Environment:** names go into `service.json` as `secrets.required`; literal values are copied to the project `.env`, which is added to `.gitignore`. Values in `.env` already are kept, and `${VAR}` values are left for you.
-- **Drops, and says so:** `command`, `env_file` and every other untranslated key; anonymous volumes and paths outside the project; `network_mode` other than `service:<name>`; services with `profiles`, `extends`, an out-of-tree build or a `${VAR}` image.
+- **Environment:** values that cannot be secrets go into the committed `service.json` `env`: numbers, `true`/`false`, another imported service's address without a user, password or query (`http://invidious:3000`, `redis:6379`), and pure `${VAR}` references. Every other name, and any name that looks like a credential (`PASSWORD`, `TOKEN`, `KEY`, ...), goes into `secrets.required`, with its literal value copied to the project `.env`, which is added to `.gitignore`. Values in `.env` already are kept.
+- **Keeps:** a `command` on an image as the `CMD` of a one-line `Dockerfile` next to `service.json` (Flisk sets no container command), a `healthcheck` as `healthCheck.command`, a published database port as `extraPorts`, and `restart: on-failure:N` as `on-failure`.
+- **Drops, and says so:** `command` on a service with a `build`, `env_file` and every other untranslated key; anonymous volumes and paths outside the project; `network_mode` other than `service:<name>`; services with `profiles`, `extends`, an out-of-tree build or a `${VAR}` image.
 - **Does not start anything.** Run `flisk up` with Flisk core **1.4.24 or later**. The importer does not check the core version.
 - **Exit codes:** `0` when the plan succeeds (`--dry-run`, files written, or all `service.json` files already exist), `1` when no root `docker-compose.yml` or mappable service is found, and `2` when the write prompt is declined or closed without an answer (`Nothing written.`).
 

package/dist/detectors/compose.js

@@ -44,6 +44,7 @@ const TRANSLATED_KEYS = new Set([
     "network_mode",
     "volumes",
     "restart",
+    "command",
 ]);
 const CAPABILITY = /^[A-Z_]+$/;
 const USER = /^[A-Za-z0-9_.:-]+$/;
@@ -211,6 +212,24 @@ function envValues(environment) {
     }
     return out;
 }
+/** A value that is nothing but Compose interpolation, such as `${URL}` or `${PORT:-80}`. Holds no secret itself. */
+const ONLY_REFERENCES = /^(\$\{[A-Za-z_][A-Za-z0-9_]*(?::?[-?+][^}\n]*)?\})+$/;
+/** Values that are only `${VAR}` references, which service.json `env` can keep as they are (Flisk 1.4.24). */
+function envReferences(environment) {
+    const pairs = Array.isArray(environment)
+        ? environment
+            .filter((e) => typeof e === "string" && e.includes("="))
+            .map((e) => [e.slice(0, e.indexOf("=")), e.slice(e.indexOf("=") + 1)])
+        : isObj(environment)
+            ? Object.entries(environment)
+            : [];
+    const out = {};
+    for (const [key, raw] of pairs) {
+        if (typeof raw === "string" && ONLY_REFERENCES.test(raw))
+            out[key] = raw;
+    }
+    return out;
+}
 function commandOf(command) {
     if (typeof command === "string")
         return command;
@@ -218,6 +237,55 @@ function commandOf(command) {
         return command.join(" ");
     return undefined;
 }
+/** Compose `command` as the argument vector the container runs; a string goes through a shell, as Compose runs it. */
+function containerCommandOf(command) {
+    if (typeof command === "string" && command.trim())
+        return ["sh", "-c", command];
+    if (Array.isArray(command) && command.length > 0 && command.every((c) => typeof c === "string"))
+        return command;
+    return undefined;
+}
+/** Seconds from a Compose duration such as `30s` or `2m`; undefined for anything it cannot read exactly. */
+function secondsOf(value) {
+    if (typeof value === "number" && value > 0)
+        return value;
+    const match = typeof value === "string" ? value.match(/^(\d+)(s|m)?$/) : null;
+    if (!match)
+        return undefined;
+    const seconds = Number(match[1]) * (match[2] === "m" ? 60 : 1);
+    return seconds > 0 ? seconds : undefined;
+}
+/** Compose `healthcheck` as a probe command (exit 0 is ready). `NONE` and `disable` give no probe. */
+function healthProbeOf(healthcheck) {
+    if (!isObj(healthcheck) || healthcheck.disable === true)
+        return undefined;
+    const test = healthcheck.test;
+    let command;
+    if (typeof test === "string")
+        command = ["sh", "-c", test];
+    else if (Array.isArray(test) && test.every((t) => typeof t === "string")) {
+        const [kind, ...rest] = test;
+        if (kind === "CMD" && rest.length > 0)
+            command = rest;
+        else if (kind === "CMD-SHELL" && rest.length > 0)
+            command = ["sh", "-c", rest.join(" ")];
+    }
+    if (!command)
+        return undefined;
+    const interval = secondsOf(healthcheck.interval);
+    const timeout = secondsOf(healthcheck.timeout);
+    const retries = typeof healthcheck.retries === "number" &&
+        Number.isInteger(healthcheck.retries) &&
+        healthcheck.retries > 0
+        ? healthcheck.retries
+        : undefined;
+    return {
+        command,
+        ...(interval ? { interval } : {}),
+        ...(timeout ? { timeout } : {}),
+        ...(retries ? { retries } : {}),
+    };
+}
 function healthPathOf(healthcheck) {
     if (!isObj(healthcheck) || !Array.isArray(healthcheck.test))
         return undefined;
@@ -237,7 +305,7 @@ function healthPathOf(healthcheck) {
 }
 registerDetector({
     id: "compose",
-    claim: (relPath) => relPath === "docker-compose.yml",
+    claim: (relPath) => /^(docker-)?compose\.ya?ml$/.test(relPath),
     parse(absPath, relPath) {
         const composeDir = dirname(relPath) === "." ? "" : dirname(relPath);
         const base = { file: relPath, detector: "compose" };
@@ -345,8 +413,11 @@ registerDetector({
                     ? Object.keys(raw.depends_on)
                     : [];
             const command = commandOf(raw.command);
+            const containerCommand = containerCommandOf(raw.command);
+            const healthProbe = healthProbeOf(raw.healthcheck);
             const envKeys = envNames(raw.environment);
             const env = envValues(raw.environment);
+            const envRefs = envReferences(raw.environment);
             const foundHealthPath = healthPathOf(raw.healthcheck);
             const healthCheckPath = foundHealthPath && HEALTH_CHECK_PATH_PATTERN.test(foundHealthPath)
                 ? foundHealthPath
@@ -365,9 +436,10 @@ registerDetector({
                 });
             }
             // Every published port Flisk does not route stays published, on 127.0.0.1 (service.json extraPorts).
+            const proxied = selectedPort !== undefined && !NON_HTTP_PORTS.has(selectedPort);
             const extraPorts = (Array.isArray(raw.ports) ? raw.ports : [])
                 .map(publishedPort)
-                .filter((p) => p !== undefined && p.port !== selectedPort);
+                .filter((p) => p !== undefined && (p.port !== selectedPort || !proxied));
             let sidecarOf;
             if (raw.network_mode !== undefined) {
                 const mode = typeof raw.network_mode === "string" ? raw.network_mode : "";
@@ -392,9 +464,19 @@ registerDetector({
                         reason: `service "${name}" volume ${JSON.stringify(entry)} is not translated (an anonymous volume, a path outside the imported directory, or Compose variables); not mounted`,
                     });
             }
-            const restart = typeof raw.restart === "string" && RESTART.has(raw.restart)
-                ? raw.restart
+            // `on-failure:5` caps the retries; Flisk has the policy without the cap.
+            const restartName = typeof raw.restart === "string"
+                ? raw.restart.replace(/^(on-failure):\d+$/, "$1")
                 : undefined;
+            const restart = restartName !== undefined && RESTART.has(restartName)
+                ? restartName
+                : undefined;
+            if (restart && restartName !== raw.restart) {
+                skips.push({
+                    ...base,
+                    reason: `service "${name}" restart ${JSON.stringify(raw.restart)} keeps "${restart}"; the retry limit is not carried over`,
+                });
+            }
             if (raw.restart !== undefined && !restart) {
                 skips.push({
                     ...base,
@@ -419,11 +501,14 @@ registerDetector({
                 ...(typeof raw.image === "string" ? { image: raw.image } : {}),
                 ...(dockerfile ? { dockerfile } : {}),
                 ...(command ? { command } : {}),
+                ...(containerCommand ? { containerCommand } : {}),
+                ...(healthProbe ? { healthCheck: healthProbe } : {}),
                 ...(selectedPort !== undefined ? { port: selectedPort } : {}),
                 ...(healthCheckPath ? { healthCheckPath } : {}),
                 exposeViaProxy,
                 ...(envKeys.length ? { envKeys } : {}),
                 ...(Object.keys(env).length ? { env } : {}),
+                ...(Object.keys(envRefs).length ? { envRefs } : {}),
                 ...(dependsOn.length ? { dependsOn } : {}),
                 ...(container ? { container } : {}),
                 ...(extraPorts.length ? { extraPorts } : {}),

package/dist/merge.js

@@ -63,6 +63,7 @@ function mergeGroup(name, dir, members, resolve) {
         kind: ordered[0].kind,
         envKeys: [...new Set(ordered.flatMap((c) => c.envKeys ?? []))].sort(),
         env: Object.assign({}, ...[...ordered].reverse().map((c) => c.env ?? {})),
+        envRefs: Object.assign({}, ...[...ordered].reverse().map((c) => c.envRefs ?? {})),
         dependsOn: [...new Set(ordered.flatMap((c) => (c.dependsOn ?? []).map(resolve)))].sort(),
         exposeViaProxy: ordered.find((c) => c.exposeViaProxy !== undefined)?.exposeViaProxy,
         // Only Compose states these, so the first source that has one wins.
@@ -70,6 +71,8 @@ function mergeGroup(name, dir, members, resolve) {
         ...firstOf(ordered, "extraPorts"),
         ...firstOf(ordered, "volumes"),
         ...firstOf(ordered, "restart"),
+        ...firstOf(ordered, "containerCommand"),
+        ...firstOf(ordered, "healthCheck"),
         ...(ordered.find((c) => c.sidecarOf)?.sidecarOf
             ? { sidecarOf: resolve(ordered.find((c) => c.sidecarOf)?.sidecarOf) }
             : {}),

package/dist/plan.js

@@ -7,7 +7,7 @@ import { selectDetectors } from "./registry.js";
 import { scanTree } from "./scan.js";
 import { findUnsupported } from "./unsupported.js";
 const COMPOSE_FILE = "docker-compose.yml";
-/** Names `docker compose` also picks up, in its own order of preference; none of them is read. */
+/** Names `docker compose` also picks up, in its own order of preference. They are read only when docker-compose.yml is absent. */
 const OTHER_COMPOSE_FILES = ["compose.yaml", "compose.yml", "docker-compose.yaml"];
 /** Files `docker compose up` merges over the main file; none of them is applied. */
 const OVERRIDE_FILES = [
@@ -16,37 +16,33 @@ const OVERRIDE_FILES = [
     "docker-compose.override.yaml",
     "docker-compose.override.yml",
 ];
-/** Read the one supported input, root docker-compose.yml. This function writes nothing. */
+/** Read the one supported input, the root Compose file. This function writes nothing. */
 export function buildPlan(dir) {
     const root = resolve(dir);
     if (!statSync(root, { throwIfNoEntry: false })?.isDirectory()) {
         throw new Error(`${dir} is not a directory`);
     }
-    const composePath = COMPOSE_FILE;
     const detectors = selectDetectors(["compose"]);
     const candidates = [];
     const skips = [];
     const isFile = (name) => statSync(resolve(root, name), { throwIfNoEntry: false })?.isFile() ?? false;
-    if (!isFile(composePath)) {
-        const others = OTHER_COMPOSE_FILES.filter(isFile);
-        throw new ImportError(`No root docker-compose.yml found in ${root}${others.length ? ` (found ${others.join(", ")})` : ""}; flisk import reads only that file`, others.length
-            ? [
-                `Other Compose file names are not read yet. To import ${others[0]}, link it: ln -s ${others[0]} docker-compose.yml`,
-            ]
-            : ["Run flisk import in the directory that holds docker-compose.yml"]);
+    const present = OTHER_COMPOSE_FILES.filter(isFile);
+    const composePath = isFile(COMPOSE_FILE) ? COMPOSE_FILE : present[0];
+    if (!composePath) {
+        throw new ImportError(`No root docker-compose.yml found in ${root} (nor compose.yaml); flisk import reads only the root Compose file`, ["Run flisk import in the directory that holds docker-compose.yml (or compose.yaml)"]);
     }
-    for (const name of OTHER_COMPOSE_FILES.filter(isFile)) {
+    for (const name of present.filter((n) => n !== composePath)) {
         skips.push({
             file: name,
             detector: "compose",
-            reason: `not read; \`docker compose\` prefers it over ${COMPOSE_FILE}, so what it runs may differ from this plan`,
+            reason: `not read; \`docker compose\` prefers it over ${composePath}, so what it runs may differ from this plan`,
         });
     }
     for (const name of OVERRIDE_FILES.filter(isFile)) {
         skips.push({
             file: name,
             detector: "compose",
-            reason: `not applied; \`docker compose up\` merges it over ${COMPOSE_FILE}, flisk import does not`,
+            reason: `not applied; \`docker compose up\` merges it over ${composePath}, flisk import does not`,
         });
     }
     for (const detector of detectors) {

package/dist/write.js

@@ -6,6 +6,31 @@ import { toResourceName } from "./merge.js";
 import { usedManifestPorts } from "./plan.js";
 /** Secret names the service schema accepts (`secrets.required`). */
 const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
+/** Names that may hold a credential whatever their value looks like; these always stay secrets. */
+const CREDENTIAL_NAME = /PASS|SECRET|TOKEN|KEY|PRIVATE|CREDENTIAL|AUTH|HMAC|SALT|CERT|SESSION|COOKIE/i;
+/**
+ * Whether a literal Compose value cannot be a secret, so it may be committed in service.json `env`: a number, true or false,
+ * or the address of another service in this import (a URL or host:port with no user, password, query or fragment).
+ */
+function isPlainValue(value, serviceHosts) {
+    if (/^-?\d+(\.\d+)?$/.test(value) || /^(true|false)$/i.test(value))
+        return true;
+    const hostPort = value.match(/^([a-z0-9][a-z0-9._-]*):(\d{1,5})$/);
+    if (hostPort)
+        return serviceHosts.has(hostPort[1]);
+    try {
+        const url = new URL(value);
+        return (serviceHosts.has(url.hostname) &&
+            !url.username &&
+            !url.password &&
+            !url.search &&
+            !url.hash &&
+            /^[A-Za-z0-9._~/-]*$/.test(url.pathname));
+    }
+    catch {
+        return false;
+    }
+}
 function readEnvKeys(path) {
     if (!existsSync(path))
         return new Set();
@@ -78,6 +103,8 @@ export function planWrites(plan, force) {
         return !reason;
     });
     const names = new Set(accepted.map((s) => s.name));
+    // Hostnames other services use for these: the resource name, which stays the container hostname, and the Compose name.
+    const serviceHosts = new Set(plan.services.flatMap((s) => (s.originalName ? [s.name, s.originalName] : [s.name])));
     for (const service of accepted) {
         const path = resolve(servicesDir, stack, service.name, "service.json");
         const exists = existsSync(path);
@@ -133,8 +160,32 @@ export function planWrites(plan, force) {
             notes.push("no health path: the service does not serve HTTP (non-HTTP image or job)");
         if (!isPrivate && !nonHttp && !service.healthCheckPath)
             notes.push("health path / is assumed: no HTTP health path could be read from Compose. Traefik routes only while it answers 2xx or 3xx");
-        // Names go into service.json as secrets, so Flisk injects them; values go to the gitignored project .env.
-        const secretNames = service.envKeys.filter((key) => ENV_NAME.test(key));
+        // A probe command is the only way to describe a non-HTTP check; a routed service keeps its HTTP path.
+        const healthCheck = service.healthCheck && !(healthCheckPath && service.healthCheckPath)
+            ? service.healthCheck
+            : undefined;
+        // Values that cannot be secrets (numbers, booleans, another service's address, pure ${VAR} references) go into the
+        // committed service.json `env` (Flisk 1.4.24). Every other name is a secret: Flisk injects it from the gitignored .env.
+        const validNames = service.envKeys.filter((key) => ENV_NAME.test(key));
+        const env = {};
+        for (const key of validNames) {
+            if (CREDENTIAL_NAME.test(key))
+                continue;
+            const literal = service.env[key];
+            const reference = service.envRefs[key];
+            if (literal !== undefined && isPlainValue(literal, serviceHosts))
+                env[key] = literal;
+            else if (literal === undefined && reference !== undefined)
+                env[key] = reference;
+        }
+        const referenced = [
+            ...new Set(Object.values(env).flatMap((value) => [...value.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)(:?-)?/g)]
+                .filter((m) => !m[2])
+                .map((m) => m[1]))),
+        ].filter((name) => !envKept.has(name) && !(name in envAdd));
+        if (referenced.length)
+            notes.push(`env reads ${referenced.join(", ")} from the project .env; add them there`);
+        const secretNames = validNames.filter((key) => !(key in env));
         const skippedNames = service.envKeys.filter((key) => !ENV_NAME.test(key));
         if (skippedNames.length)
             notes.push(`environment names not written (Flisk accepts A-Z, 0-9 and _): ${skippedNames.join(", ")}`);
@@ -154,12 +205,27 @@ export function planWrites(plan, force) {
             notes.push(`image "${service.image}" is not written: with build and image, Flisk builds from ${service.dockerfile} and does not pull the tag`);
         // With both build and image, Compose builds the Dockerfile and tags the result as `image`.
         // Omit `image` from service.json so Flisk builds from the Dockerfile instead of pulling the tag.
-        const source = service.dockerfile
-            ? {
-                buildContext: relative(dirname(path), resolve(plan.root, service.dir)).split("\\").join("/") || ".",
-                dockerfile: service.dockerfile,
-            }
-            : { image: service.image };
+        // Flisk has no container `command`, so a Compose `command` on an image becomes the CMD of a one-line Dockerfile.
+        const files = [];
+        const wrapWithCommand = !service.dockerfile && service.image && service.containerCommand;
+        if (wrapWithCommand) {
+            files.push({
+                path: resolve(dirname(path), "Dockerfile"),
+                content: `# From the Compose \`command\` of this service (flisk import). Flisk sets no container command, so it lives here.\nFROM ${service.image}\nCMD ${JSON.stringify(service.containerCommand)}\n`,
+            });
+            notes.push(`command ${JSON.stringify(service.containerCommand)} is written as the CMD of a Dockerfile on ${service.image}`);
+        }
+        else if (service.dockerfile && service.containerCommand) {
+            notes.push(`command ${JSON.stringify(service.containerCommand)} not written: set it as CMD in ${service.dockerfile}`);
+        }
+        const source = wrapWithCommand
+            ? { buildContext: ".", dockerfile: "Dockerfile" }
+            : service.dockerfile
+                ? {
+                    buildContext: relative(dirname(path), resolve(plan.root, service.dir)).split("\\").join("/") || ".",
+                    dockerfile: service.dockerfile,
+                }
+                : { image: service.image };
         // A sidecar shares another written service's network; one whose host is not written stays a normal service.
         const sidecarOf = service.sidecarOf && service.sidecarOf !== service.name && names.has(service.sidecarOf)
             ? service.sidecarOf
@@ -193,9 +259,13 @@ export function planWrites(plan, force) {
             stack,
             port,
             ...(healthCheckPath ? { healthCheckPath } : {}),
+            ...(healthCheck ? { healthCheck } : {}),
             ...source,
             ...(isPrivate ? { exposeViaProxy: false } : {}),
             ...(dependsOn.length ? { dependsOn } : {}),
+            ...(Object.keys(env).length
+                ? { env: Object.fromEntries(Object.entries(env).sort(([a], [b]) => a.localeCompare(b))) }
+                : {}),
             ...(secretNames.length ? { secrets: { required: secretNames } } : {}),
             ...(service.restart ? { restart: service.restart } : {}),
             ...(service.container ? { container: service.container } : {}),
@@ -211,7 +281,7 @@ export function planWrites(plan, force) {
                 }
                 : {}),
         };
-        items.push({ service, path, manifest, status, notes });
+        items.push({ service, path, manifest, files, status, notes });
     }
     return {
         projectRoot,
@@ -233,6 +303,8 @@ export function applyWrites(writes) {
             continue;
         mkdirSync(dirname(item.path), { recursive: true });
         writeFileSync(item.path, `${JSON.stringify(item.manifest, null, 2)}\n`);
+        for (const file of item.files)
+            writeFileSync(file.path, file.content);
         written.push(item);
     }
     const keys = Object.keys(writes.env.add);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flisk-import",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Import a repository's root docker-compose.yml into Flisk service.json files, or register its Tiltfile.",
   "type": "module",
   "license": "MIT",