@yadurajfleetos/cli 0.5.0 → 0.6.0

package/dist/commands/services.js

@@ -403,6 +403,31 @@ export const initCommand = {
         const name = (typeof flags.name === 'string' ? flags.name : '') ||
             process.cwd().split('/').pop()?.toLowerCase().replace(/[^a-z0-9-]+/g, '-') ||
             'app';
+        // Read the whole repository first. A monorepo, an apps/ directory, or a
+        // service beside a database is the ordinary case, and describing only the
+        // current directory left every one of those to be written out by hand.
+        const { discover, manifestFromDiscovery } = await import('../discover.js');
+        const found = await discover();
+        if (found.services.length > 1 || found.databases.length) {
+            const { manifest, questions } = manifestFromDiscovery(found, {
+                fleet: typeof flags.fleet === 'string' ? flags.fleet : undefined,
+                node: typeof flags.node === 'string' ? flags.node : undefined,
+            });
+            await writeFile(path, manifest);
+            console.log(`${c.green('created')} ${path}`);
+            if (found.layout)
+                console.log(c.dim(`  ${found.layout}`));
+            for (const s of found.services) {
+                console.log(c.dim(`  · ${s.name}  ${s.dir}  ${s.detection.label}`));
+            }
+            for (const db of found.databases) {
+                console.log(c.dim(`  · ${db.name} (${db.engine}) — ${db.because}`));
+            }
+            for (const q of questions)
+                console.log(`  ${c.yellow('?')} ${q}`);
+            console.log(c.dim(`\n  check it with: fleet apply --dry-run`));
+            return;
+        }
         const d = await detect();
         // Write a Dockerfile if we generated one and none exists.
         if (d.dockerfile) {

package/dist/compose.js

@@ -1,4 +1,5 @@
 import { parse as parseYaml } from 'yaml';
+import { safeDatabaseName } from './dbnames.js';
 /** Images Fleet manages as databases rather than as plain containers. */
 const ENGINE_IMAGES = {
     postgres: 'postgres',
@@ -184,13 +185,22 @@ export function composeToFleet(source, opts = {}) {
     const databases = [];
     /** Compose names that became databases, so `depends_on` can point at them. */
     const asDatabase = new Set();
+    /** Compose name -> the name the database is declared under, which differs
+        whenever the compose name would collide with its engine's own env vars. */
+    const dbNames = new Map();
+    const takenDbNames = new Set();
     // Databases first, so a service's `uses` can reference one by name.
     for (const [name, raw] of Object.entries(servicesRaw)) {
         const svc = (asRecord(raw) ?? {});
         const image = typeof svc.image === 'string' ? svc.image : '';
         const engine = image ? engineFor(image) : null;
-        if (engine)
-            asDatabase.add(name);
+        if (!engine)
+            continue;
+        asDatabase.add(name);
+        // Named here, in the pre-pass, not where the database is rendered: a
+        // service declared before it in the file still has to point at the name it
+        // ends up with, and the main loop would not know it yet.
+        dbNames.set(name, safeDatabaseName(name, engine, takenDbNames));
     }
     for (const [name, raw] of Object.entries(servicesRaw)) {
         const svc = (asRecord(raw) ?? {});
@@ -199,7 +209,11 @@ export function composeToFleet(source, opts = {}) {
         if (asDatabase.has(name)) {
             const engine = engineFor(image);
             const major = versionOf(image);
-            const lines = [`  ${name}:`];
+            const declared = dbNames.get(name);
+            if (declared !== name) {
+                notes.push(`${name} is declared as "${declared}": a database named after its own engine derives a password secret that collides with the engine's own environment variable, and the manifest is rejected.`);
+            }
+            const lines = [`  ${declared}:`];
             lines.push(`    engine: ${major ? `${engine}@${major}` : engine}`);
             if (dbNode)
                 lines.push(`    node: ${scalar(dbNode)}`);
@@ -295,7 +309,7 @@ export function composeToFleet(source, opts = {}) {
         const deps = Array.isArray(svc.depends_on)
             ? svc.depends_on.filter((d) => typeof d === 'string')
             : Object.keys(asRecord(svc.depends_on) ?? {});
-        const dbDeps = deps.filter((d) => asDatabase.has(d));
+        const dbDeps = deps.filter((d) => asDatabase.has(d)).map((d) => dbNames.get(d) ?? d);
         const svcDeps = deps.filter((d) => !asDatabase.has(d));
         if (dbDeps.length)
             lines.push(`    uses: [${dbDeps.join(', ')}]`);

package/dist/dbnames.js

@@ -0,0 +1,34 @@
+/**
+ * What to call a managed database, when the obvious name breaks.
+ *
+ * The manifest derives a database's password secret from its name:
+ * `passwordRefFor("main")` is MAIN_PASSWORD. That is a good rule, and it has
+ * one collision. A database called `postgres` derives POSTGRES_PASSWORD, which
+ * is also the environment variable the Postgres image itself expects — so the
+ * generated service ends up with the same key in both `env` and `secrets`, and
+ * the parser rejects the whole manifest.
+ *
+ * It matters because `postgres:` is one of the most common service names in a
+ * docker-compose file, so the most ordinary input produced something the
+ * product refused. Renaming to `db` is also simply what a person writing this
+ * by hand would have done: `uses: [db, cache]` reads better than
+ * `uses: [postgres, redis]`.
+ */
+/** What a person would call each engine, rather than what the image is called. */
+const FRIENDLY = {
+    postgres: 'db',
+    mysql: 'db',
+    mariadb: 'db',
+    mongo: 'db',
+    redis: 'cache',
+};
+export function safeDatabaseName(preferred, engine, taken) {
+    const base = preferred.toLowerCase() === engine.toLowerCase() ? (FRIENDLY[engine] ?? 'db') : preferred;
+    // Two Postgres databases in one file both want to be `db`.
+    let name = base;
+    let n = 2;
+    while (taken.has(name))
+        name = `${base}${n++}`;
+    taken.add(name);
+    return name;
+}

package/dist/discover.js

@@ -0,0 +1,349 @@
+import { readdir, readFile, access } from 'node:fs/promises';
+import { basename, join, relative } from 'node:path';
+import { detect } from './detect.js';
+import { safeDatabaseName } from './dbnames.js';
+const exists = async (p) => {
+    try {
+        await access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+};
+const readText = async (p) => {
+    try {
+        return await readFile(p, 'utf8');
+    }
+    catch {
+        return '';
+    }
+};
+const readJson = async (p) => {
+    try {
+        return JSON.parse(await readFile(p, 'utf8'));
+    }
+    catch {
+        return null;
+    }
+};
+/** Directories that are never a service, whatever else they contain. */
+const IGNORED = new Set([
+    'node_modules', '.git', 'dist', 'build', 'target', 'vendor', '.next',
+    'coverage', '__pycache__', '.venv', 'venv', 'tmp', '.turbo', '.cache',
+]);
+/**
+ * Dependencies that mean "this service talks to a database".
+ *
+ * A driver in the dependency list is evidence, not a guess: nobody installs
+ * `pg` for a service that does not speak to Postgres.
+ */
+const DB_HINTS = [
+    {
+        engine: 'postgres',
+        deps: [
+            'pg', 'postgres', 'pg-promise', 'postgres.js', '@prisma/client', 'prisma',
+            'typeorm', 'sequelize', 'drizzle-orm', 'knex', 'psycopg2', 'psycopg2-binary',
+            'psycopg', 'asyncpg', 'sqlalchemy', 'django', 'lib/pq', 'jackc/pgx', 'sqlx',
+            'tokio-postgres', 'diesel',
+        ],
+    },
+    { engine: 'redis', deps: ['redis', 'ioredis', 'redis-py', 'go-redis', 'bull', 'bullmq', 'celery'] },
+    { engine: 'mysql', deps: ['mysql', 'mysql2', 'mysqlclient', 'pymysql', 'go-sql-driver'] },
+    { engine: 'mongo', deps: ['mongodb', 'mongoose', 'pymongo', 'motor', 'mongo-driver'] },
+];
+/** Dependencies that mean "this wants a GPU and a lot more memory". */
+const GPU_HINTS = [
+    'torch', 'pytorch', 'tensorflow', 'transformers', 'vllm', 'accelerate',
+    'onnxruntime-gpu', 'jax', 'diffusers', 'sentence-transformers', 'llama-cpp-python',
+];
+const SECRET_HINT = /(PASSWORD|SECRET|TOKEN|APIKEY|API_KEY|ACCESS_KEY|PRIVATE|CREDENTIAL|_DSN|SALT|CERT|_URL$)/i;
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** Every dependency name a directory declares, across ecosystems. */
+async function dependenciesOf(dir) {
+    const out = [];
+    const pkg = await readJson(join(dir, 'package.json'));
+    if (pkg) {
+        for (const field of ['dependencies', 'devDependencies', 'peerDependencies']) {
+            const deps = pkg[field];
+            if (deps && typeof deps === 'object')
+                out.push(...Object.keys(deps));
+        }
+    }
+    // Python: one name per line, before any version specifier.
+    for (const file of ['requirements.txt', 'requirements-prod.txt', 'pyproject.toml', 'Pipfile']) {
+        const text = await readText(join(dir, file));
+        for (const line of text.split('\n')) {
+            const name = line.trim().split(/[=<>!~\[;#"']/)[0]?.trim();
+            if (name && /^[A-Za-z][A-Za-z0-9._-]*$/.test(name))
+                out.push(name.toLowerCase());
+        }
+    }
+    // Go and Rust name their dependencies as paths and table keys respectively;
+    // a substring match on the whole file is enough to spot a driver.
+    for (const file of ['go.mod', 'go.sum', 'Cargo.toml']) {
+        const text = await readText(join(dir, file));
+        if (text)
+            out.push(...text.split('\n').map((l) => l.trim().toLowerCase()));
+    }
+    return out;
+}
+const mentions = (deps, needle) => deps.some((d) => d === needle || d.includes(needle));
+/** Variable names a service declares it needs. */
+async function envFrom(dir) {
+    const env = [];
+    const secrets = [];
+    for (const file of ['.env.example', '.env.sample', '.env.template']) {
+        const text = await readText(join(dir, file));
+        for (const line of text.split('\n')) {
+            const trimmed = line.trim();
+            if (!trimmed || trimmed.startsWith('#'))
+                continue;
+            const key = trimmed.split('=')[0]?.trim();
+            if (!key || !ENV_NAME.test(key))
+                continue;
+            // A credential and a setting are different things: one belongs in the
+            // manifest, the other must never appear in a file anyone commits.
+            if (SECRET_HINT.test(key))
+                secrets.push(key);
+            else
+                env.push(key);
+        }
+    }
+    return { env: [...new Set(env)], secrets: [...new Set(secrets)] };
+}
+/** Workspace globs, from whichever convention this repository uses. */
+async function workspaceGlobs(root) {
+    const pkg = await readJson(join(root, 'package.json'));
+    const ws = pkg?.workspaces;
+    if (Array.isArray(ws))
+        return { layout: 'npm workspaces', globs: ws.filter((w) => typeof w === 'string') };
+    if (ws && typeof ws === 'object' && Array.isArray(ws.packages)) {
+        return { layout: 'yarn workspaces', globs: ws.packages };
+    }
+    const pnpm = await readText(join(root, 'pnpm-workspace.yaml'));
+    if (pnpm) {
+        const globs = pnpm
+            .split('\n')
+            .map((l) => l.trim())
+            .filter((l) => l.startsWith('- '))
+            .map((l) => l.slice(2).replace(/['"]/g, '').trim())
+            .filter(Boolean);
+        if (globs.length)
+            return { layout: 'pnpm workspace', globs };
+    }
+    const lerna = await readJson(join(root, 'lerna.json'));
+    if (Array.isArray(lerna?.packages))
+        return { layout: 'lerna', globs: lerna.packages };
+    const goWork = await readText(join(root, 'go.work'));
+    if (goWork) {
+        const globs = [...goWork.matchAll(/^\s*\.?\/?([\w./-]+)\s*$/gm)]
+            .map((m) => m[1])
+            .filter((p) => p !== 'go' && !p.includes('use') && p !== '.');
+        if (globs.length)
+            return { layout: 'go workspace', globs };
+    }
+    const cargo = await readText(join(root, 'Cargo.toml'));
+    const members = cargo.match(/members\s*=\s*\[([^\]]*)\]/);
+    if (members) {
+        const globs = [...members[1].matchAll(/["']([^"']+)["']/g)].map((m) => m[1]);
+        if (globs.length)
+            return { layout: 'cargo workspace', globs };
+    }
+    return null;
+}
+/**
+ * Expand the one glob shape workspaces actually use: a directory then `/*`.
+ *
+ * Deliberately not a glob library. `apps/*` and `packages/*` are what these
+ * files contain in practice, and a dependency to handle the rest is not worth
+ * carrying in a CLI whose whole install is one package.
+ */
+async function expand(root, globs) {
+    const dirs = new Set();
+    for (const glob of globs) {
+        if (!glob.includes('*')) {
+            if (await exists(join(root, glob)))
+                dirs.add(glob);
+            continue;
+        }
+        const base = glob.slice(0, glob.indexOf('*')).replace(/\/$/, '');
+        try {
+            for (const entry of await readdir(join(root, base), { withFileTypes: true })) {
+                if (entry.isDirectory() && !IGNORED.has(entry.name))
+                    dirs.add(join(base, entry.name));
+            }
+        }
+        catch {
+            /* a workspace glob pointing at nothing is the repository's problem */
+        }
+    }
+    return [...dirs];
+}
+/** Conventional layouts, for repositories that declare no workspaces. */
+async function conventionalDirs(root) {
+    const dirs = [];
+    for (const parent of ['apps', 'services', 'packages']) {
+        try {
+            for (const entry of await readdir(join(root, parent), { withFileTypes: true })) {
+                if (entry.isDirectory() && !IGNORED.has(entry.name))
+                    dirs.push(join(parent, entry.name));
+            }
+        }
+        catch {
+            /* not this layout */
+        }
+    }
+    return dirs;
+}
+/**
+ * Whether a directory is something to deploy.
+ *
+ * A monorepo is mostly libraries, and a manifest that tries to deploy a shared
+ * types package is noise the reader has to delete. Something to run says so:
+ * it has a Dockerfile, a start script, or a recognised framework.
+ */
+async function isDeployable(dir, d) {
+    if (d.hasDockerfile)
+        return true;
+    if (d.framework !== 'unknown') {
+        const pkg = await readJson(join(dir, 'package.json'));
+        if (pkg) {
+            const scripts = (pkg.scripts ?? {});
+            // A package with no way to start is a library, whatever it depends on.
+            if (!scripts.start && !scripts.dev && !scripts.serve && d.framework !== 'vite')
+                return false;
+        }
+        return true;
+    }
+    return false;
+}
+const serviceName = (dir, root) => {
+    const raw = dir === '.' ? basename(root) : basename(dir);
+    return raw.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '') || 'app';
+};
+export async function discover(root = process.cwd()) {
+    const notes = [];
+    const ws = await workspaceGlobs(root);
+    let candidates = [];
+    let layout = null;
+    if (ws) {
+        layout = ws.layout;
+        candidates = await expand(root, ws.globs);
+        notes.push(`${ws.layout} declares ${candidates.length} package${candidates.length === 1 ? '' : 's'}`);
+    }
+    else {
+        const conventional = await conventionalDirs(root);
+        if (conventional.length) {
+            layout = 'apps/ and services/ directories';
+            candidates = conventional;
+            notes.push(`no workspace file, but ${conventional.length} directories look like services`);
+        }
+    }
+    // Always consider the root itself: a repository can be a monorepo and still
+    // deploy something from its top level.
+    if (!candidates.includes('.'))
+        candidates.unshift('.');
+    const services = [];
+    const allDeps = new Set();
+    for (const dir of candidates) {
+        const abs = dir === '.' ? root : join(root, dir);
+        const d = await detect(abs);
+        const deps = await dependenciesOf(abs);
+        deps.forEach((x) => allDeps.add(x));
+        // Which engines THIS package depends on. Taking the union across the
+        // repository made a Next.js frontend declare it uses the database, which
+        // is a claim the manifest should not be making on its behalf.
+        const ownEngines = DB_HINTS.filter(({ deps: hints }) => hints.some((h) => mentions(deps, h))).map((x) => x.engine);
+        if (!(await isDeployable(abs, d)))
+            continue;
+        // A root that only exists to hold workspaces is not a service; without
+        // this a monorepo gains a phantom service named after the repository.
+        if (dir === '.' && ws && !d.hasDockerfile)
+            continue;
+        const { env, secrets } = await envFrom(abs);
+        const gpu = GPU_HINTS.find((h) => mentions(deps, h)) ?? null;
+        services.push({
+            name: serviceName(dir, root),
+            dir: dir === '.' ? '.' : `./${relative(root, abs)}`,
+            detection: d,
+            env,
+            secrets,
+            gpu: gpu ? `depends on ${gpu}` : null,
+            engines: ownEngines,
+            // A model needs room; everything else gets a modest default the reader
+            // can lower once they know what it actually uses.
+            ramMb: gpu ? 4096 : 512,
+        });
+    }
+    const deps = [...allDeps];
+    const databases = [];
+    const takenDbNames = new Set();
+    for (const { engine, deps: hints } of DB_HINTS) {
+        const hit = hints.find((h) => mentions(deps, h));
+        // Named `db` and `cache` rather than `postgres` and `redis`: a database
+        // named after its engine collides with that engine's own environment
+        // variables, and it is not what anyone would write by hand either.
+        if (hit) {
+            databases.push({
+                name: safeDatabaseName(engine, engine, takenDbNames),
+                engine,
+                because: `something depends on ${hit}`,
+            });
+        }
+    }
+    if (!services.length) {
+        notes.push('nothing deployable found — no Dockerfile, and no framework this recognises');
+    }
+    for (const db of databases)
+        notes.push(`${db.engine}: ${db.because}`);
+    return { root, layout, services, databases, notes };
+}
+/** Render a discovery as a manifest. */
+export function manifestFromDiscovery(d, opts = {}) {
+    const questions = [];
+    const fleet = opts.fleet ?? 'homelab';
+    const lines = [`fleet: ${fleet}`, '', 'services:'];
+    d.services.forEach((s, i) => {
+        if (i)
+            lines.push('');
+        lines.push(`  ${s.name}:`);
+        lines.push(`    build: ${s.dir}`);
+        lines.push('    placement: flexible');
+        if (s.detection.port !== 80)
+            lines.push(`    container_port: ${s.detection.port}`);
+        lines.push(`    resources: { ram: ${s.ramMb}Mi, cpu: 0.5 }`);
+        lines.push(`    health: { path: ${s.detection.healthPath} }`);
+        if (s.gpu) {
+            lines.push('    gpu: true');
+            questions.push(`${s.name}: ${s.gpu}, so it asks for a GPU — remove "gpu: true" if it runs on CPU.`);
+        }
+        if (s.env.length) {
+            lines.push('    env:');
+            for (const key of s.env)
+                lines.push(`      ${key}: ""   # from .env.example`);
+        }
+        if (s.secrets.length)
+            lines.push(`    secrets: [${s.secrets.join(', ')}]`);
+        const mine = d.databases.filter((db) => s.engines.includes(db.engine));
+        if (mine.length)
+            lines.push(`    uses: [${mine.map((db) => db.name).join(', ')}]`);
+    });
+    if (d.databases.length) {
+        lines.push('', 'databases:');
+        d.databases.forEach((db, i) => {
+            if (i)
+                lines.push('');
+            lines.push(`  ${db.name}:`);
+            lines.push(`    engine: ${db.engine}`);
+            if (opts.node)
+                lines.push(`    node: ${opts.node}`);
+            else {
+                lines.push('    node: CHANGE_ME');
+                questions.push(`${db.name}: a database must name the node that holds its data — replace CHANGE_ME, or re-run with --node.`);
+            }
+            lines.push('    backup: daily');
+        });
+    }
+    return { manifest: lines.join('\n') + '\n', questions };
+}

package/dist/index.js

@@ -16,7 +16,7 @@ const GROUPS = [
         'getting started',
         [
             ['up [service]', 'Deploy the whole fleet.yaml, in dependency order'],
-            ['init', 'Scaffold a fleet.yaml and Dockerfile from this repository'],
+            ['init', 'Read the repository — monorepo, databases, secrets — and write a fleet.yaml'],
             ['import [file]', 'Convert a docker-compose.yml into a fleet.yaml'],
             ['config show', 'Show the saved control plane and selected fleet'],
             ['use <fleet>', 'Select the default fleet for later commands'],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yadurajfleetos/cli",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Fleet OS command-line interface for deploying and orchestrating services on user-owned hardware",
   "type": "module",
   "license": "MIT",