flisk-import 0.4.1 → 0.6.0

package/README.md

@@ -13,9 +13,10 @@ flisk import . --yes         # write service.json files and project setup
 - **Reads one file:** the root `docker-compose.yml`. Not `compose.yaml`, override files, nested Compose files, Dockerfiles on their own, `package.json`, or Procfiles.
 - **Writes:** one bring-your-own `service.json` per Compose service, plus `.flisk/project.json` if the repo is not a Flisk project yet. It never edits the Compose file or overwrites a `service.json` without `--force`.
 - **Keeps:** the service names (`db` stays `db`, so it stays the hostname), the build context and Dockerfile (or the image), the container port as written (3000, 80, 5432), `depends_on`, and an HTTP health path. Databases and brokers are private, with no proxy route.
-- **Environment:** names go into `service.json` as `secrets.required`; literal values are copied to the project `.env`, which is added to `.gitignore`. Values in `.env` already are kept, and `${VAR}` values are left for you.
-- **Drops, and says so:** volumes, `command`, `env_file` and every other untranslated key; services with `profiles`, `extends`, an out-of-tree build or a `${VAR}` image.
-- **Does not start anything.** Run `flisk up` with Flisk core **1.4.11 or later**. The importer does not check the core version.
+- **Runtime keys:** named volumes and bind mounts inside the project (`volumes`), `cap_add`, `cap_drop`, `init`, `read_only`, `user`, `security_opt` and `shm_size` (`container`), extra published ports (`extraPorts`, on 127.0.0.1 only), `network_mode: service:<name>` (`sidecarOf`) and `restart`.
+- **Environment:** values that cannot be secrets go into the committed `service.json` `env`: numbers, `true`/`false`, another imported service's address without a user, password or query (`http://invidious:3000`, `redis:6379`), and pure `${VAR}` references. Every other name, and any name that looks like a credential (`PASSWORD`, `TOKEN`, `KEY`, ...), goes into `secrets.required`, with its literal value copied to the project `.env`, which is added to `.gitignore`. Values in `.env` already are kept.
+- **Drops, and says so:** `command`, `env_file` and every other untranslated key; anonymous volumes and paths outside the project; `network_mode` other than `service:<name>`; services with `profiles`, `extends`, an out-of-tree build or a `${VAR}` image.
+- **Does not start anything.** Run `flisk up` with Flisk core **1.4.24 or later**. The importer does not check the core version.
 - **Exit codes:** `0` when the plan succeeds (`--dry-run`, files written, or all `service.json` files already exist), `1` when no root `docker-compose.yml` or mappable service is found, and `2` when the write prompt is declined or closed without an answer (`Nothing written.`).
 
 See [docs/import.md](docs/import.md) for the full mapping and limits. `flisk import` is the core CLI command backed by this package.

package/dist/detectors/compose.js

@@ -22,8 +22,10 @@ const NON_HTTP_PORTS = new Set([
 ]);
 /**
  * Service keys this detector translates. Every other key is named in the report: Flisk runs a
- * bring-your-own container without volumes, command overrides, env files and the rest.
- * `healthcheck` only contributes an HTTP path; `environment` only contributes names.
+ * bring-your-own container without command overrides, env files and the rest.
+ * `healthcheck` only contributes an HTTP path; `environment` only contributes names. The runtime keys
+ * (capabilities, volumes, network_mode, restart) map onto service.json fields; a value that cannot be
+ * carried over is named on its own line instead.
  */
 const TRANSLATED_KEYS = new Set([
     "image",
@@ -32,7 +34,155 @@ const TRANSLATED_KEYS = new Set([
     "environment",
     "depends_on",
     "healthcheck",
+    "cap_add",
+    "cap_drop",
+    "init",
+    "read_only",
+    "user",
+    "security_opt",
+    "shm_size",
+    "network_mode",
+    "volumes",
+    "restart",
 ]);
+const CAPABILITY = /^[A-Z_]+$/;
+const USER = /^[A-Za-z0-9_.:-]+$/;
+const SECURITY_OPT = /^[A-Za-z0-9_.:=/-]+$/;
+const VOLUME_NAME = /^[a-z0-9][a-z0-9_-]*$/;
+const CONTAINER_PATH = /^\/[A-Za-z0-9_./-]*$/;
+const HOST_PATH = /^[A-Za-z0-9_./-]+$/;
+const RESTART = new Set(["no", "on-failure", "unless-stopped", "always"]);
+/** Compose `shm_size` as a size string such as 256mb; a number is bytes. */
+function shmSizeOf(value) {
+    if (typeof value === "number" && Number.isInteger(value) && value > 0)
+        return `${value}b`;
+    if (typeof value === "string" && /^[0-9]+([bkmg]b?)?$/i.test(value))
+        return value;
+    return undefined;
+}
+/** `container` for service.json, and the values that could not be carried over. */
+function containerOf(raw) {
+    const container = {};
+    const dropped = [];
+    for (const [key, field] of [
+        ["cap_add", "capAdd"],
+        ["cap_drop", "capDrop"],
+    ]) {
+        if (raw[key] === undefined)
+            continue;
+        const caps = Array.isArray(raw[key]) ? raw[key] : [];
+        const kept = caps.filter((c) => typeof c === "string" && CAPABILITY.test(c));
+        if (kept.length !== caps.length || caps.length === 0)
+            dropped.push(key);
+        if (kept.length)
+            container[field] = kept;
+    }
+    if (raw.init !== undefined) {
+        if (typeof raw.init === "boolean")
+            container.init = raw.init;
+        else
+            dropped.push("init");
+    }
+    if (raw.read_only !== undefined) {
+        if (typeof raw.read_only === "boolean")
+            container.readOnly = raw.read_only;
+        else
+            dropped.push("read_only");
+    }
+    if (raw.user !== undefined) {
+        const user = typeof raw.user === "number" ? String(raw.user) : raw.user;
+        if (typeof user === "string" && USER.test(user))
+            container.user = user;
+        else
+            dropped.push("user");
+    }
+    if (raw.security_opt !== undefined) {
+        const opts = Array.isArray(raw.security_opt) ? raw.security_opt : [];
+        const kept = opts.filter((o) => typeof o === "string" && SECURITY_OPT.test(o));
+        if (kept.length !== opts.length || opts.length === 0)
+            dropped.push("security_opt");
+        if (kept.length)
+            container.securityOpt = kept;
+    }
+    if (raw.shm_size !== undefined) {
+        const size = shmSizeOf(raw.shm_size);
+        if (size)
+            container.shmSize = size;
+        else
+            dropped.push("shm_size");
+    }
+    return { ...(Object.keys(container).length ? { container } : {}), dropped };
+}
+/** One published port: container port, host port when Compose names a literal one, and protocol. */
+function publishedPort(entry) {
+    const port = containerPort(entry);
+    if (port === undefined)
+        return undefined;
+    let host;
+    let protocol = "tcp";
+    if (isObj(entry)) {
+        const published = typeof entry.published === "string" ? Number(entry.published) : entry.published;
+        if (typeof published === "number" && Number.isInteger(published))
+            host = published;
+        if (entry.protocol === "udp")
+            protocol = "udp";
+    }
+    else if (typeof entry === "string") {
+        const [mapping, proto] = entry.split("/");
+        if (proto === "udp")
+            protocol = "udp";
+        const parts = (mapping ?? "").split(":");
+        const hostPart = parts.length >= 2 ? parts[parts.length - 2] : undefined;
+        if (hostPart && /^\d+$/.test(hostPart))
+            host = Number.parseInt(hostPart, 10);
+    }
+    return {
+        port,
+        ...(host !== undefined && host >= 1 && host <= 65535 && host !== port
+            ? { hostPort: host }
+            : {}),
+        ...(protocol === "udp" ? { protocol } : {}),
+    };
+}
+/** One `volumes:` entry. Bind sources are resolved against the Compose file's directory. */
+function volumeOf(entry, composeDir) {
+    let source;
+    let target;
+    let readOnly = false;
+    let type;
+    if (typeof entry === "string") {
+        const parts = entry.split(":");
+        if (parts.length < 2 || parts.length > 3)
+            return undefined;
+        [source, target] = parts;
+        readOnly = parts[2] === "ro";
+        if (parts[2] !== undefined && parts[2] !== "ro" && parts[2] !== "rw")
+            return undefined;
+    }
+    else if (isObj(entry)) {
+        ({ source, target, type } = entry);
+        readOnly = entry.read_only === true;
+        if (type !== undefined && type !== "volume" && type !== "bind")
+            return undefined;
+    }
+    else {
+        return undefined;
+    }
+    if (typeof source !== "string" || typeof target !== "string" || !CONTAINER_PATH.test(target))
+        return undefined;
+    const bind = type === "bind" || source.startsWith(".") || source.startsWith("/");
+    if (!bind) {
+        return VOLUME_NAME.test(source)
+            ? { source, target, ...(readOnly ? { readOnly } : {}), bind }
+            : undefined;
+    }
+    if (isAbsolute(source) || !HOST_PATH.test(source))
+        return undefined;
+    const resolved = normalize(join(composeDir, source));
+    if (resolved === ".." || resolved.startsWith("../"))
+        return undefined;
+    return { source: resolved, target, ...(readOnly ? { readOnly } : {}), bind };
+}
 function envNames(environment) {
     if (Array.isArray(environment)) {
         return environment
@@ -61,6 +211,24 @@ function envValues(environment) {
     }
     return out;
 }
+/** A value that is nothing but Compose interpolation, such as `${URL}` or `${PORT:-80}`. Holds no secret itself. */
+const ONLY_REFERENCES = /^(\$\{[A-Za-z_][A-Za-z0-9_]*(?::?[-?+][^}\n]*)?\})+$/;
+/** Values that are only `${VAR}` references, which service.json `env` can keep as they are (Flisk 1.4.24). */
+function envReferences(environment) {
+    const pairs = Array.isArray(environment)
+        ? environment
+            .filter((e) => typeof e === "string" && e.includes("="))
+            .map((e) => [e.slice(0, e.indexOf("=")), e.slice(e.indexOf("=") + 1)])
+        : isObj(environment)
+            ? Object.entries(environment)
+            : [];
+    const out = {};
+    for (const [key, raw] of pairs) {
+        if (typeof raw === "string" && ONLY_REFERENCES.test(raw))
+            out[key] = raw;
+    }
+    return out;
+}
 function commandOf(command) {
     if (typeof command === "string")
         return command;
@@ -180,13 +348,13 @@ registerDetector({
             if (ports.length > 1 && selectedPort === undefined) {
                 skips.push({
                     ...base,
-                    reason: `service "${name}" lists several ports (${ports.join(", ")}); no single HTTP container port could be identified`,
+                    reason: `service "${name}" lists several ports (${ports.join(", ")}); no single HTTP container port could be identified, so none is routed (all are published on 127.0.0.1 as extraPorts)`,
                 });
             }
             else if (ports.length > 1) {
                 skips.push({
                     ...base,
-                    reason: `service "${name}" publishes extra container ports (${ports.filter((port) => port !== selectedPort).join(", ")}); Flisk routes only ${selectedPort}`,
+                    reason: `service "${name}" publishes extra container ports (${ports.filter((port) => port !== selectedPort).join(", ")}); Flisk routes only ${selectedPort} and publishes the others on 127.0.0.1 as extraPorts`,
                 });
             }
             const dependsOn = Array.isArray(raw.depends_on)
@@ -197,6 +365,7 @@ registerDetector({
             const command = commandOf(raw.command);
             const envKeys = envNames(raw.environment);
             const env = envValues(raw.environment);
+            const envRefs = envReferences(raw.environment);
             const foundHealthPath = healthPathOf(raw.healthcheck);
             const healthCheckPath = foundHealthPath && HEALTH_CHECK_PATH_PATTERN.test(foundHealthPath)
                 ? foundHealthPath
@@ -207,6 +376,50 @@ registerDetector({
                     reason: `service "${name}" health path "${foundHealthPath}" has characters Flisk does not accept (only a-z, 0-9, / and -); not used`,
                 });
             }
+            const { container, dropped } = containerOf(raw);
+            for (const key of dropped) {
+                skips.push({
+                    ...base,
+                    reason: `service "${name}" ${key} ${JSON.stringify(raw[key])} has values Flisk does not accept; not used`,
+                });
+            }
+            // Every published port Flisk does not route stays published, on 127.0.0.1 (service.json extraPorts).
+            const extraPorts = (Array.isArray(raw.ports) ? raw.ports : [])
+                .map(publishedPort)
+                .filter((p) => p !== undefined && p.port !== selectedPort);
+            let sidecarOf;
+            if (raw.network_mode !== undefined) {
+                const mode = typeof raw.network_mode === "string" ? raw.network_mode : "";
+                if (mode.startsWith("service:") && mode.length > "service:".length) {
+                    sidecarOf = mode.slice("service:".length);
+                }
+                else {
+                    skips.push({
+                        ...base,
+                        reason: `service "${name}" network_mode "${mode}" is not translated; Flisk only shares another service's network (service:<name>)`,
+                    });
+                }
+            }
+            const volumes = [];
+            for (const entry of Array.isArray(raw.volumes) ? raw.volumes : []) {
+                const volume = volumeOf(entry, composeDir);
+                if (volume)
+                    volumes.push(volume);
+                else
+                    skips.push({
+                        ...base,
+                        reason: `service "${name}" volume ${JSON.stringify(entry)} is not translated (an anonymous volume, a path outside the imported directory, or Compose variables); not mounted`,
+                    });
+            }
+            const restart = typeof raw.restart === "string" && RESTART.has(raw.restart)
+                ? raw.restart
+                : undefined;
+            if (raw.restart !== undefined && !restart) {
+                skips.push({
+                    ...base,
+                    reason: `service "${name}" restart ${JSON.stringify(raw.restart)} is not a policy Flisk accepts; not used`,
+                });
+            }
             const untranslated = Object.keys(raw).filter((key) => !TRANSLATED_KEYS.has(key));
             if (untranslated.length) {
                 skips.push({
@@ -230,7 +443,13 @@ registerDetector({
                 exposeViaProxy,
                 ...(envKeys.length ? { envKeys } : {}),
                 ...(Object.keys(env).length ? { env } : {}),
+                ...(Object.keys(envRefs).length ? { envRefs } : {}),
                 ...(dependsOn.length ? { dependsOn } : {}),
+                ...(container ? { container } : {}),
+                ...(extraPorts.length ? { extraPorts } : {}),
+                ...(sidecarOf ? { sidecarOf } : {}),
+                ...(volumes.length ? { volumes } : {}),
+                ...(restart ? { restart } : {}),
                 confidence: "high",
                 sources: [base],
             });

package/dist/flisk.js

@@ -9,7 +9,7 @@ export const SERVICE_MANIFEST_SCHEMA_URL = "https://flisk.sh/schema.service.json
  * release after 1.4.10 (flisk#2646) accepts a bring-your-own port outside 4000-5999 and two-letter
  * names, which this importer now writes. Stated, not checked: it does not read the installed version.
  */
-export const MIN_CORE_VERSION = "1.4.11";
+export const MIN_CORE_VERSION = "1.4.24";
 /**
  * The core change that runs a registered Tiltfile (`orchestration.source: "tiltfile"`) from `flisk up`. Stated, not
  * checked, like MIN_CORE_VERSION.

package/dist/merge.js

@@ -49,6 +49,10 @@ const SCALAR_FIELDS = [
 function rank(candidate) {
     return Math.min(...candidate.sources.map((s) => precedence(s.detector)));
 }
+function firstOf(ordered, key) {
+    const holder = ordered.find((c) => c[key] !== undefined);
+    return holder ? { [key]: holder[key] } : {};
+}
 function mergeGroup(name, dir, members, resolve) {
     const ordered = [...members].sort((a, b) => rank(a) - rank(b));
     const originalName = ordered.find((c) => c.name !== undefined && c.name !== name)?.name;
@@ -59,8 +63,17 @@ function mergeGroup(name, dir, members, resolve) {
         kind: ordered[0].kind,
         envKeys: [...new Set(ordered.flatMap((c) => c.envKeys ?? []))].sort(),
         env: Object.assign({}, ...[...ordered].reverse().map((c) => c.env ?? {})),
+        envRefs: Object.assign({}, ...[...ordered].reverse().map((c) => c.envRefs ?? {})),
         dependsOn: [...new Set(ordered.flatMap((c) => (c.dependsOn ?? []).map(resolve)))].sort(),
         exposeViaProxy: ordered.find((c) => c.exposeViaProxy !== undefined)?.exposeViaProxy,
+        // Only Compose states these, so the first source that has one wins.
+        ...firstOf(ordered, "container"),
+        ...firstOf(ordered, "extraPorts"),
+        ...firstOf(ordered, "volumes"),
+        ...firstOf(ordered, "restart"),
+        ...(ordered.find((c) => c.sidecarOf)?.sidecarOf
+            ? { sidecarOf: resolve(ordered.find((c) => c.sidecarOf)?.sidecarOf) }
+            : {}),
         procfileCommands: ordered.flatMap((c) => c.sources.some((source) => source.detector === "procfile") && c.command ? [c.command] : []),
         conflicts: [],
         sources: [],
@@ -104,6 +117,7 @@ export function mergeCandidates(candidates, rootName) {
     const table = assignResourceNames([
         ...candidates.flatMap((c) => (c.name === undefined ? [] : [c.name])),
         ...candidates.flatMap((c) => c.dependsOn ?? []),
+        ...candidates.flatMap((c) => (c.sidecarOf ? [c.sidecarOf] : [])),
         ...candidates.flatMap((c) => (c.name === undefined ? [basename(c.dir) || rootName] : [])),
     ]);
     const resolve = (raw) => table.get(raw) ?? toResourceName(raw);

package/dist/write.js

@@ -6,6 +6,31 @@ import { toResourceName } from "./merge.js";
 import { usedManifestPorts } from "./plan.js";
 /** Secret names the service schema accepts (`secrets.required`). */
 const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
+/** Names that may hold a credential whatever their value looks like; these always stay secrets. */
+const CREDENTIAL_NAME = /PASS|SECRET|TOKEN|KEY|PRIVATE|CREDENTIAL|AUTH|HMAC|SALT|CERT|SESSION|COOKIE/i;
+/**
+ * Whether a literal Compose value cannot be a secret, so it may be committed in service.json `env`: a number, true or false,
+ * or the address of another service in this import (a URL or host:port with no user, password, query or fragment).
+ */
+function isPlainValue(value, serviceHosts) {
+    if (/^-?\d+(\.\d+)?$/.test(value) || /^(true|false)$/i.test(value))
+        return true;
+    const hostPort = value.match(/^([a-z0-9][a-z0-9._-]*):(\d{1,5})$/);
+    if (hostPort)
+        return serviceHosts.has(hostPort[1]);
+    try {
+        const url = new URL(value);
+        return (serviceHosts.has(url.hostname) &&
+            !url.username &&
+            !url.password &&
+            !url.search &&
+            !url.hash &&
+            /^[A-Za-z0-9._~/-]*$/.test(url.pathname));
+    }
+    catch {
+        return false;
+    }
+}
 function readEnvKeys(path) {
     if (!existsSync(path))
         return new Set();
@@ -78,6 +103,8 @@ export function planWrites(plan, force) {
         return !reason;
     });
     const names = new Set(accepted.map((s) => s.name));
+    // Hostnames other services use for these: the resource name, which stays the container hostname, and the Compose name.
+    const serviceHosts = new Set(plan.services.flatMap((s) => (s.originalName ? [s.name, s.originalName] : [s.name])));
     for (const service of accepted) {
         const path = resolve(servicesDir, stack, service.name, "service.json");
         const exists = existsSync(path);
@@ -133,8 +160,28 @@ export function planWrites(plan, force) {
             notes.push("no health path: the service does not serve HTTP (non-HTTP image or job)");
         if (!isPrivate && !nonHttp && !service.healthCheckPath)
             notes.push("health path / is assumed: no HTTP health path could be read from Compose. Traefik routes only while it answers 2xx or 3xx");
-        // Names go into service.json as secrets, so Flisk injects them; values go to the gitignored project .env.
-        const secretNames = service.envKeys.filter((key) => ENV_NAME.test(key));
+        // Values that cannot be secrets (numbers, booleans, another service's address, pure ${VAR} references) go into the
+        // committed service.json `env` (Flisk 1.4.24). Every other name is a secret: Flisk injects it from the gitignored .env.
+        const validNames = service.envKeys.filter((key) => ENV_NAME.test(key));
+        const env = {};
+        for (const key of validNames) {
+            if (CREDENTIAL_NAME.test(key))
+                continue;
+            const literal = service.env[key];
+            const reference = service.envRefs[key];
+            if (literal !== undefined && isPlainValue(literal, serviceHosts))
+                env[key] = literal;
+            else if (literal === undefined && reference !== undefined)
+                env[key] = reference;
+        }
+        const referenced = [
+            ...new Set(Object.values(env).flatMap((value) => [...value.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)(:?-)?/g)]
+                .filter((m) => !m[2])
+                .map((m) => m[1]))),
+        ].filter((name) => !envKept.has(name) && !(name in envAdd));
+        if (referenced.length)
+            notes.push(`env reads ${referenced.join(", ")} from the project .env; add them there`);
+        const secretNames = validNames.filter((key) => !(key in env));
         const skippedNames = service.envKeys.filter((key) => !ENV_NAME.test(key));
         if (skippedNames.length)
             notes.push(`environment names not written (Flisk accepts A-Z, 0-9 and _): ${skippedNames.join(", ")}`);
@@ -160,6 +207,31 @@ export function planWrites(plan, force) {
                 dockerfile: service.dockerfile,
             }
             : { image: service.image };
+        // A sidecar shares another written service's network; one whose host is not written stays a normal service.
+        const sidecarOf = service.sidecarOf && service.sidecarOf !== service.name && names.has(service.sidecarOf)
+            ? service.sidecarOf
+            : undefined;
+        if (service.sidecarOf && !sidecarOf)
+            notes.push(`network_mode not written: "${service.sidecarOf}" is not in this import`);
+        const extraPorts = sidecarOf ? [] : (service.extraPorts ?? []);
+        if (sidecarOf && service.extraPorts?.length)
+            notes.push(`ports not written: a sidecar has none of its own; publish them on "${sidecarOf}"`);
+        if (extraPorts.length)
+            notes.push(`published on 127.0.0.1 only: ${extraPorts.map((p) => `${p.hostPort ?? p.port}->${p.port}/${p.protocol ?? "tcp"}`).join(", ")}`);
+        // Bind mounts are relative to the service directory in service.json, and must stay inside the project.
+        const serviceDir = dirname(path);
+        const volumes = (service.volumes ?? []).flatMap((volume) => {
+            if (!volume.bind)
+                return [volume];
+            const absolute = resolve(plan.root, volume.source);
+            const fromProject = relative(projectRoot, absolute).split("\\").join("/");
+            if (fromProject === ".." || fromProject.startsWith("../")) {
+                notes.push(`volume ${volume.source}:${volume.target} not written: it is outside the project`);
+                return [];
+            }
+            const rel = relative(serviceDir, absolute).split("\\").join("/") || ".";
+            return [{ ...volume, source: rel.startsWith(".") ? rel : `./${rel}` }];
+        });
         const manifest = {
             $schema: SERVICE_MANIFEST_SCHEMA_URL,
             schemaVersion: SERVICE_MANIFEST_SCHEMA_VERSION,
@@ -171,7 +243,23 @@ export function planWrites(plan, force) {
             ...source,
             ...(isPrivate ? { exposeViaProxy: false } : {}),
             ...(dependsOn.length ? { dependsOn } : {}),
+            ...(Object.keys(env).length
+                ? { env: Object.fromEntries(Object.entries(env).sort(([a], [b]) => a.localeCompare(b))) }
+                : {}),
             ...(secretNames.length ? { secrets: { required: secretNames } } : {}),
+            ...(service.restart ? { restart: service.restart } : {}),
+            ...(service.container ? { container: service.container } : {}),
+            ...(extraPorts.length ? { extraPorts } : {}),
+            ...(sidecarOf ? { sidecarOf } : {}),
+            ...(volumes.length
+                ? {
+                    volumes: volumes.map(({ source, target, readOnly }) => ({
+                        source,
+                        target,
+                        ...(readOnly ? { readOnly } : {}),
+                    })),
+                }
+                : {}),
         };
         items.push({ service, path, manifest, status, notes });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flisk-import",
-  "version": "0.4.1",
+  "version": "0.6.0",
   "description": "Import a repository's root docker-compose.yml into Flisk service.json files, or register its Tiltfile.",
   "type": "module",
   "license": "MIT",