@git.zone/cli 2.22.0 → 2.24.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@git.zone/cli",
   "private": false,
-  "version": "2.22.0",
+  "version": "2.24.0",
   "description": "A comprehensive CLI tool for enhancing and managing local development workflows with gitzone utilities, focusing on project setup, version control, code formatting, and template management.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -40,7 +40,7 @@
   "homepage": "https://code.foss.global/git.zone/cli#README",
   "devDependencies": {
     "@git.zone/tsbuild": "^4.4.2",
-    "@git.zone/tsrun": "^2.0.4",
+    "@git.zone/tsrun": "^2.0.5",
     "@git.zone/tstest": "^3.6.6",
     "@types/node": "^25.9.3"
   },
@@ -90,7 +90,8 @@
     "last 1 chrome versions"
   ],
   "scripts": {
-    "test": "(pnpm run clean && pnpm run prepareTest && pnpm run testCli && pnpm run testFormat && pnpm run testCommit && pnpm run testDeprecate && pnpm run testVersion && pnpm run testReadme && pnpm run testUpdate && pnpm run testTemplateNpm && pnpm run testTemplateLit) && rm -rf .nogit/test",
+    "test": "(pnpm run testUnit && pnpm run clean && pnpm run prepareTest && pnpm run testCli && pnpm run testFormat && pnpm run testCommit && pnpm run testDeprecate && pnpm run testVersion && pnpm run testReadme && pnpm run testUpdate && pnpm run testTemplateNpm && pnpm run testTemplateLit) && rm -rf .nogit/test",
+    "testUnit": "tstest 'test.*.node.ts'",
     "build": "tsbuild tsfolders",
     "clean": "(rm -rf .nogit/test/)",
     "prepareTest": "(mkdir -p .nogit && git clone https://gitlab.com/sandboxzone/sandbox-npmts.git .nogit/test/)",

package/readme.hints.md

@@ -127,6 +127,94 @@ The commit module now supports `-y/--yes` flag for non-interactive commits:
 - Fully backward compatible with interactive mode
 - CI/CD friendly for automated workflows
 
+## Services Module (`mod_services`) — Cleanup Design
+
+### Why data accumulated
+
+`gitzone services clean` never actually worked for MongoDB or MinIO. Service
+containers write as their own uid — mongod as 999 with `drwx------` on
+subdirectories, MinIO as root under `.minio.sys` — so a recursive delete run as
+the invoking user removed only the part of the tree it owned. For MongoDB the
+result was a half-deleted WiredTiger dataset that crash-looped on the next start
+(`WiredTigerHS.wt file is corrupted or missing`), kept alive indefinitely by
+`restart: unless-stopped`. Users hit a failing `clean`, gave up, and the data
+stayed. This was the root cause of multi-GB accumulation, not a missing feature.
+
+`DockerContainer.removeDataDirectory` fixes it: try a native delete, and if the
+directory survives, empty it from a short-lived `--user 0` container that mounts
+only that directory. It verifies the directory is gone and throws otherwise — a
+partial delete is worse than no delete.
+
+### Identifying tool-owned resources
+
+Follows the `git.zone.*` label convention already established by
+`@git.zone/tsdocker`. Containers created by `gitzone services` carry:
+
+```text
+git.zone.tool=gitzone-services
+git.zone.service=mongodb|minio|elasticsearch
+git.zone.project-path=<abs path>
+git.zone.data-path=<abs path>
+git.zone.safe-to-prune=true
+```
+
+Data directories are additionally claimed by a marker at
+`<project>/.nogit/.gitzone-services.json`. The marker deliberately lives outside
+the bind-mounted directory so it can never confuse mongod, MinIO or
+Elasticsearch at runtime.
+
+Nothing is ever matched by image or by bare name pattern. Containers predating
+labels are only identified when exactly one registry entry claims their name.
+
+### Prune safety model (`classes.servicepruner.ts`)
+
+Mirrors `TsDockerPruner`: build a plan, print it, and mutate only on `--apply`
+with every item re-verified immediately before removal.
+
+Projects classify as `live`, `stale`, `orphaned`, or `unknown`. Only `stale` and
+`orphaned` are candidates; `unknown` never is. Ambiguity must *tighten* the
+decision — a container name claimed by two projects forces both to `unknown`,
+because an unattributable container also means "no container is running" cannot
+be proven. Getting this backwards initially made 1.18 GB look reclaimable when
+it was not.
+
+Invariants, enforced at plan time and again at apply time:
+
+- A data directory is never removed while any running container bind-mounts it
+  or an overlapping path (`helpers.pathsOverlap`).
+- A path must match `<project>/.nogit/{mongodata,miniodata,esdata}` exactly
+  (`isSafeServiceDataPath`); this is an allowlist, checked after resolution.
+- Ownership must be proven by marker or registry entry.
+- Staleness is re-checked at apply time, so a project started between plan and
+  apply aborts the removal.
+- An unreachable Docker daemon disables all container and data reclamation, so
+  ambiguity never reads as absence.
+
+### MongoDB auth modes
+
+Default is authenticated, single-node replica set with a keyfile.
+`gitzone services auth mongodb off` opts into a no-auth instance for runtimes
+that cannot complete a SCRAM handshake over `node:crypto` (Deno). In that mode
+the port is published on `127.0.0.1` only — publishing scope is the real
+exposure control, since `--bind_ip_all` binds inside the container netns — and a
+non-local `MONGODB_HOST` is refused. `MONGO_INITDB_ROOT_*` must be omitted in
+no-auth mode because the official entrypoint turns them into `--auth`.
+
+Re-enabling auth over data created without it finds no root user, because
+`MONGO_INITDB_ROOT_*` only applies to an empty dbpath. `ensureMongoRootUser`
+bootstraps it through MongoDB's localhost exception, which by design only works
+while zero users exist and therefore cannot escalate anything.
+
+### Coupling: services config is a deployment input
+
+`@git.zone/cli.services` is read by `@git.zone/tsdeploy`
+(`deriveRequiredCapabilities` in `classes.cloudlydeployment.ts`) to derive a
+workload's `requiredCapabilities`. It **throws** unless the value is a flat array
+of unique, non-empty, lowercase canonical strings. Never add sub-keys under it
+and never change the persisted names — turning it into an object would fail real
+deployments. Any new services configuration belongs in `.nogit/env.json` or a
+sibling key.
+
 ## Development Tips
 
 - Always check readme.plan.md for ongoing improvement plans

package/readme.md

@@ -446,20 +446,86 @@ gitzone services set mongodb,minio
 # Check status
 gitzone services status
 
+# Machine-readable status, including connection strings and data sizes
+gitzone services status --json
+
 # Print MongoDB Compass connection string
 gitzone services compass
 
 # Show logs
 gitzone services logs mongo 50
+```
+
+Service config is stored in `.nogit/env.json`. Data is stored below `.nogit/`, so it stays out of Git.
+
+### Consuming a service programmatically
 
-# Stop containers but keep data
-gitzone services stop
+`gitzone services status --json` emits only JSON on stdout, so a test suite or
+script can read a live connection string without parsing human output:
 
-# Remove containers and data
-gitzone services clean
+```bash
+gitzone services status --json | jq -r '.services.mongodb.connectionString'
 ```
 
-Service config is stored in `.nogit/env.json`. Data is stored below `.nogit/`, so it stays out of Git.
+### Cleanup levels
+
+Cleanup is tiered, from fully resumable to irreversible:
+
+| Command | Containers | Data | Notes |
+| --- | --- | --- | --- |
+| `gitzone services stop` | kept (stopped) | kept | fully resumable |
+| `gitzone services remove` | removed | kept | resumable; `--yes` to skip the prompt |
+| `gitzone services clean` | removed | **removed** | irreversible; needs a typed `yes` or `--yes` |
+| `gitzone services prune` | see below | see below | machine-wide; dry run unless `--apply` |
+
+`clean` and `prune` remove data written by the container user (mongod runs as
+uid 999, MinIO as root) by escalating to a short-lived privileged container
+scoped to that one directory. They either remove a directory completely or fail
+— a partial delete would leave a corrupt database behind.
+
+### Reclaiming space across projects
+
+Service data is per project and survives container removal, so it accumulates.
+`gitzone services prune` reports what every registered project holds and what
+can be reclaimed. It is read-only unless `--apply` is passed:
+
+```bash
+# Report only: what exists, what is reclaimable, and why
+gitzone services prune
+
+# Change the inactivity threshold (default 30 days)
+gitzone services prune --stale-days 90
+
+# Actually reclaim, non-interactively
+gitzone services prune --apply --yes
+```
+
+A project is only a candidate when there is positive evidence it is finished
+with: its directory is gone, or it has been inactive past the threshold with no
+container running. Anything ambiguous — an unlabeled container claimed by more
+than one project, an unreachable Docker daemon, a directory still mounted by a
+running container — is reported and skipped rather than reclaimed. Containers
+are identified by the `git.zone.tool=gitzone-services` label or by an
+unambiguous registry claim, never by image or name pattern, so pruning cannot
+touch containers created by anything else.
+
+### MongoDB authentication
+
+MongoDB runs as a single-node replica set with authentication enabled, so
+multi-document transactions work. Authentication can be disabled per project for
+runtimes whose `node:crypto` cannot complete a SCRAM handshake (notably Deno):
+
+```bash
+gitzone services auth mongodb off
+gitzone services start mongo
+```
+
+This is opt-in and never implicit. With authentication disabled the database is
+published on `127.0.0.1` only, and the combination of no authentication with a
+non-local `MONGODB_HOST` is refused outright. Transactions continue to work, and
+`gitzone services status` reports the mode. Re-enabling authentication over data
+created without it bootstraps the configured root user through MongoDB's
+localhost exception.
 
 ## Templates
 

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/cli',
-  version: '2.22.0',
+  version: '2.24.0',
   description: 'A comprehensive CLI tool for enhancing and managing local development workflows with gitzone utilities, focusing on project setup, version control, code formatting, and template management.'
 }

package/ts/mod_services/classes.dockercontainer.ts

@@ -10,25 +10,57 @@ export interface IDockerRunOptions {
   ports?: { [key: string]: string };
   volumes?: { [key: string]: string };
   environment?: { [key: string]: string };
+  labels?: { [key: string]: string };
   restart?: string;
   command?: string;
 }
 
+/** Container facts needed to decide, safely, whether a container may be removed. */
+export interface IContainerInspectInfo {
+  id: string;
+  name: string;
+  state: string;
+  running: boolean;
+  labels: { [key: string]: string };
+  /** Resolved host-side sources of every bind mount. */
+  mountSources: string[];
+}
+
+/** Single-quote a value for safe interpolation into a bash command. */
+export const shellQuote = (valueArg: string): string => `'${valueArg.replace(/'/g, `'"'"'`)}'`;
+
 export class DockerContainer {
   private smartshell: plugins.smartshell.Smartshell;
-  
+  /**
+   * Wall-clock bound for docker queries and in-container commands.
+   *
+   * smartshell only arms a timeout when one is passed, so without this a wedged
+   * `docker exec` never settles and retry loops that look bounded ("30
+   * attempts") hang forever. Mirrors the `timeout Ns docker …` approach used by
+   * @git.zone/tsdocker.
+   */
+  private queryTimeoutSeconds = 60;
+  /** Longer bound for privileged cleanup, which may delete a large tree. */
+  private cleanupTimeoutSeconds = 600;
+
   constructor() {
     this.smartshell = new plugins.smartshell.Smartshell({
       executor: 'bash',
     });
   }
-  
+
+  /** `docker` prefixed with a hard wall-clock bound. */
+  private bounded(seconds: number): string {
+    return `timeout ${seconds}s docker`;
+  }
+
   /**
    * Check if Docker is installed and available
    */
   public async checkDocker(): Promise<boolean> {
     try {
-      const result = await this.smartshell.exec('docker --version');
+      // Silent: this must not print to stdout, which would corrupt --json output.
+      const result = await this.smartshell.execSilent('docker --version');
       return result.exitCode === 0;
     } catch (error) {
       return false;
@@ -120,17 +152,25 @@ export class DockerContainer {
     // Add volumes
     if (options.volumes) {
       for (const [hostPath, containerPath] of Object.entries(options.volumes)) {
-        command += ` -v "${hostPath}:${containerPath}"`;
+        command += ` -v ${shellQuote(`${hostPath}:${containerPath}`)}`;
       }
     }
-    
-    // Add environment variables
+
+    // Add environment variables. Values come from .nogit/env.json, which is
+    // user-editable, so they must not be interpolated unquoted.
     if (options.environment) {
       for (const [key, value] of Object.entries(options.environment)) {
-        command += ` -e ${key}="${value}"`;
+        command += ` -e ${shellQuote(`${key}=${value}`)}`;
       }
     }
-    
+
+    // Add labels (used to identify tool-owned containers during prune)
+    if (options.labels) {
+      for (const [key, value] of Object.entries(options.labels)) {
+        command += ` --label ${shellQuote(`${key}=${value}`)}`;
+      }
+    }
+
     // Add restart policy
     if (options.restart) {
       command += ` --restart ${options.restart}`;
@@ -154,12 +194,124 @@ export class DockerContainer {
     }
   }
   
+  /**
+   * Run a one-off command in a temporary container (docker run --rm)
+   */
+  public async runOneOff(options: {
+    image: string;
+    command: string;
+    volumes?: { [key: string]: string };
+    user?: string;
+    /** Shell used inside the container. `sh` is available in every service image. */
+    shell?: string;
+  }): Promise<boolean> {
+    let command = `${this.bounded(this.cleanupTimeoutSeconds)} run --rm`;
+    if (options.user) {
+      command += ` --user ${shellQuote(options.user)}`;
+    }
+    if (options.volumes) {
+      for (const [hostPath, containerPath] of Object.entries(options.volumes)) {
+        // Quoted: this path reaches a root `rm -rf`, so a project directory
+        // containing `$(...)`, a backtick or a quote must not reach the shell
+        // unescaped. The path allowlist constrains shape, not metacharacters.
+        command += ` -v ${shellQuote(`${hostPath}:${containerPath}`)}`;
+      }
+    }
+    // The image entrypoint must be bypassed: service images start their daemon.
+    command += ` --entrypoint ${shellQuote(options.shell || 'bash')} ${shellQuote(options.image)}`;
+    command += ` -c ${shellQuote(options.command)}`;
+    try {
+      const result = await this.smartshell.exec(command);
+      return result.exitCode === 0;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      logger.log('error', `Failed to run one-off container: ${errorMessage}`);
+      return false;
+    }
+  }
+
+  /**
+   * True when an image is already present locally.
+   */
+  public async imageExists(imageArg: string): Promise<boolean> {
+    try {
+      const result = await this.smartshell.execSilent(
+        `${this.bounded(this.queryTimeoutSeconds)} image inspect ${shellQuote(imageArg)}`,
+      );
+      return result.exitCode === 0;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  /**
+   * Remove a service data directory, escalating to a root one-off container
+   * when the invoking user cannot delete the contents.
+   *
+   * Service containers write as their own uid (mongod as 999, MinIO as root)
+   * with restrictive modes, so a plain recursive delete only removes part of the
+   * tree. For MongoDB that left a half-deleted, corrupt WiredTiger dataset that
+   * crash-looped on the next start — so a partial delete is worse than none.
+   * This either fully removes the directory or throws.
+   *
+   * The one-off container mounts only the target directory, so `rm -rf` inside
+   * it cannot reach anything else on the host.
+   */
+  public async removeDataDirectory(hostPathArg: string, imageArg: string): Promise<void> {
+    const hostPath = plugins.path.resolve(hostPathArg);
+    if (!(await plugins.smartfs.directory(hostPath).exists())) {
+      return;
+    }
+
+    // Fast path: the current user may already own everything.
+    try {
+      await plugins.smartfs.directory(hostPath).recursive().delete();
+    } catch (error) {
+      // fall through to privileged removal
+    }
+    if (!(await plugins.smartfs.directory(hostPath).exists())) {
+      return;
+    }
+
+    if (!(await this.imageExists(imageArg))) {
+      throw new Error(
+        `Cannot remove ${hostPath}: its contents belong to the container user and image ` +
+          `${imageArg} is not available locally to perform a privileged cleanup. ` +
+          `Pull ${imageArg} and retry, or remove the directory manually as root.`,
+      );
+    }
+
+    // Empty the mounted directory from inside a root container, then remove the
+    // now-empty directory from the host.
+    const emptied = await this.runOneOff({
+      image: imageArg,
+      shell: 'sh',
+      command: 'rm -rf /gitzone-target/..?* /gitzone-target/.[!.]* /gitzone-target/*',
+      volumes: { [hostPath]: '/gitzone-target' },
+      user: '0',
+    });
+    if (!emptied) {
+      throw new Error(`Privileged cleanup of ${hostPath} failed`);
+    }
+
+    try {
+      await plugins.smartfs.directory(hostPath).recursive().delete();
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      throw new Error(`Could not remove ${hostPath} after privileged cleanup: ${errorMessage}`);
+    }
+
+    if (await plugins.smartfs.directory(hostPath).exists()) {
+      throw new Error(`Refusing to report success: ${hostPath} still exists`);
+    }
+  }
+
   /**
    * Execute a command in a running container
    */
   public async exec(containerName: string, command: string): Promise<string> {
     try {
-      const result = await this.smartshell.exec(`docker exec ${containerName} ${command}`);
+      const result = await this.smartshell.exec(`${this.bounded(this.queryTimeoutSeconds)} exec ${containerName} ${command}`);
       if (result.exitCode === 0) {
         return result.stdout;
       }
@@ -169,6 +321,28 @@ export class DockerContainer {
     }
   }
   
+  /**
+   * Execute a command in a running container, returning stdout and stderr
+   * together along with the exit code.
+   *
+   * `exec` discards output on a non-zero exit and drops stderr entirely, which
+   * hides diagnostics that a caller may need to branch on.
+   */
+  public async execCombined(
+    containerName: string,
+    command: string,
+  ): Promise<{ exitCode: number; output: string }> {
+    try {
+      const result = await this.smartshell.execSilent(
+        `${this.bounded(this.queryTimeoutSeconds)} exec ${containerName} ${command} 2>&1`,
+      );
+      return { exitCode: result.exitCode, output: result.stdout || '' };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      return { exitCode: -1, output: errorMessage };
+    }
+  }
+
   /**
    * Get container logs
    */
@@ -260,4 +434,90 @@ export class DockerContainer {
       return null;
     }
   }
+
+  /**
+   * Run a `docker ps` query and return matching container ids.
+   *
+   * Throws when Docker itself fails, so callers can fail closed instead of
+   * mistaking an unreachable daemon for "no containers exist".
+   */
+  public async listIds(filterArgs: string[], includeStopped: boolean = true): Promise<string[]> {
+    const filters = filterArgs.map((filter) => `--filter ${shellQuote(filter)}`).join(' ');
+    const allFlag = includeStopped ? '-a' : '';
+    const result = await this.smartshell.execSilent(
+      `${this.bounded(this.queryTimeoutSeconds)} ps ${allFlag} ${filters} --format '{{.ID}}'`,
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(result.stderr || result.stdout || 'docker ps failed');
+    }
+    if (!result.stdout.trim()) {
+      return [];
+    }
+    return result.stdout
+      .trim()
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter(Boolean);
+  }
+
+  /**
+   * Inspect many containers at once and normalise the parts prune relies on.
+   * Throws on Docker failure so ambiguity never reads as absence.
+   */
+  public async inspectMany(idsArg: string[]): Promise<IContainerInspectInfo[]> {
+    if (idsArg.length === 0) {
+      return [];
+    }
+    const result = await this.smartshell.execSilent(
+      `${this.bounded(this.queryTimeoutSeconds)} inspect ${idsArg.map((id) => shellQuote(id)).join(' ')}`,
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(result.stderr || result.stdout || 'docker inspect failed');
+    }
+    let parsed: any;
+    try {
+      parsed = JSON.parse(result.stdout);
+    } catch (error) {
+      throw new Error('docker inspect returned unparseable output');
+    }
+    if (!Array.isArray(parsed)) {
+      throw new Error('docker inspect returned unexpected output');
+    }
+    return parsed.map((container: any) => {
+      const labels = (container?.Config?.Labels || {}) as { [key: string]: string };
+      const mountSources: string[] = [];
+      for (const mount of container?.Mounts || []) {
+        if (mount?.Source) {
+          mountSources.push(plugins.path.resolve(mount.Source));
+        }
+      }
+      return {
+        id: String(container?.Id || ''),
+        name: String(container?.Name || '').replace(/^\//, ''),
+        state: String(container?.State?.Status || 'unknown'),
+        running: container?.State?.Running === true,
+        labels,
+        mountSources,
+      };
+    });
+  }
+
+  /**
+   * Resolved host paths bind-mounted by every *currently running* container on
+   * the daemon, regardless of which tool created it.
+   *
+   * This is the primitive that makes data reclamation safe: a directory that is
+   * mounted by anything running is never a deletion candidate.
+   */
+  public async listRunningMountSources(): Promise<string[]> {
+    const ids = await this.listIds([], false);
+    const containers = await this.inspectMany(ids);
+    const sources = new Set<string>();
+    for (const container of containers) {
+      for (const source of container.mountSources) {
+        sources.add(source);
+      }
+    }
+    return [...sources];
+  }
 }

package/ts/mod_services/classes.globalregistry.ts

@@ -170,6 +170,33 @@ export class GlobalRegistry {
     return { stopped, failed };
   }
 
+  /**
+   * Cheap staleness summary for a non-blocking hint after ordinary commands.
+   *
+   * Deliberately filesystem-only: no Docker calls and no directory sizing, so
+   * it stays fast enough to run after every `start` or `status`.
+   */
+  public async getStaleSummary(
+    staleDaysArg: number,
+  ): Promise<{ orphaned: number; stale: number }> {
+    const projects = await this.getAllProjects();
+    const thresholdMs = staleDaysArg * 24 * 60 * 60 * 1000;
+    let orphaned = 0;
+    let stale = 0;
+
+    for (const [projectPath, project] of Object.entries(projects)) {
+      if (!(await plugins.smartfs.directory(projectPath).exists())) {
+        orphaned++;
+        continue;
+      }
+      if (Date.now() - project.lastActive > thresholdMs) {
+        stale++;
+      }
+    }
+
+    return { orphaned, stale };
+  }
+
   /**
    * Remove stale registry entries (projects that no longer exist on disk)
    */