dsh-scnet 0.4.2 → 0.5.0

package/skills/scnet-hpc/scripts/scnet_backends/profile.py

@@ -0,0 +1,78 @@
+"""Read the repository's simple shell-style cluster profiles safely."""
+
+from __future__ import annotations
+
+import re
+import shlex
+from pathlib import Path
+
+from .base import BackendError
+
+
+_ASSIGNMENT = re.compile(r"^([A-Z][A-Z0-9_]*)=(.*)$")
+
+
+def parse_profile(path: Path) -> dict[str, str]:
+    values: dict[str, str] = {}
+    logical_lines: list[tuple[int, str]] = []
+    pending = ""
+    pending_line = 0
+    for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not pending:
+            pending_line = line_number
+        if raw_line.endswith("\\"):
+            pending += raw_line[:-1]
+            continue
+        logical_lines.append((pending_line, pending + raw_line))
+        pending = ""
+    if pending:
+        raise BackendError(f"{path}:{pending_line}: unfinished line continuation")
+
+    for line_number, raw_line in logical_lines:
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        match = _ASSIGNMENT.match(line)
+        if not match:
+            raise BackendError(f"{path}:{line_number}: unsupported profile syntax")
+        key, raw_value = match.groups()
+        lexer = shlex.shlex(raw_value, posix=True)
+        lexer.whitespace_split = True
+        lexer.commenters = "#"
+        parts = list(lexer)
+        if len(parts) > 1:
+            raise BackendError(f"{path}:{line_number}: profile value must be scalar")
+        values[key] = parts[0] if parts else ""
+    return values
+
+
+def list_profiles(repo_root: Path) -> list[dict[str, str]]:
+    cluster_dir = repo_root / "clusters"
+    result: list[dict[str, str]] = []
+    for path in sorted(cluster_dir.glob("*.conf")):
+        if path.name.startswith("_"):
+            continue
+        profile = parse_profile(path)
+        result.append(
+            {
+                "name": path.stem,
+                "cluster_id": profile.get("CLUSTER_ID", path.stem),
+                "description": profile.get("CLUSTER_DESC", ""),
+                "default_backend": profile.get("DEFAULT_BACKEND", "ssh"),
+            }
+        )
+    return result
+
+
+def load_profile(repo_root: Path, name: str | None) -> tuple[str | None, dict[str, str]]:
+    profiles = list_profiles(repo_root)
+    if name:
+        path = repo_root / "clusters" / f"{name}.conf"
+        if not path.is_file():
+            available = ", ".join(item["name"] for item in profiles) or "(none)"
+            raise BackendError(f"unknown cluster profile {name!r}; available: {available}")
+        return name, parse_profile(path)
+    if len(profiles) == 1:
+        selected = profiles[0]["name"]
+        return selected, parse_profile(repo_root / "clusters" / f"{selected}.conf")
+    return None, {}

package/skills/scnet-hpc/scripts/scnet_backends/ssh.py

@@ -0,0 +1,258 @@
+"""SSH/Slurm backend."""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+from pathlib import Path
+from typing import Any, Mapping
+
+from .base import Backend, BackendError, require_option
+from .profile import list_profiles
+
+
+def _split_pipe(line: str, fields: list[str]) -> dict[str, str]:
+    values = line.rstrip("\n").split("|")
+    values += [""] * max(0, len(fields) - len(values))
+    return dict(zip(fields, values))
+
+
+class SSHBackend(Backend):
+    name = "ssh"
+    capabilities = frozenset(
+        {
+            "clusters",
+            "queues",
+            "limits",
+            "job",
+            "logs",
+            "submit",
+            "cancel",
+            "upload",
+            "download",
+            "exec",
+        }
+    )
+
+    def _alias(self) -> str:
+        alias = self.context.profile.get("CLUSTER_ID")
+        if not alias:
+            raise BackendError("SSH operations require --cluster and a CLUSTER_ID profile field")
+        return alias
+
+    def _run(self, argv: list[str], *, input_text: str | None = None) -> str:
+        try:
+            completed = subprocess.run(
+                argv,
+                input=input_text,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=self.context.timeout,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            raise BackendError(str(exc)) from exc
+        if completed.returncode != 0:
+            detail = completed.stderr.strip() or completed.stdout.strip()
+            raise BackendError(
+                f"command failed with exit {completed.returncode}: {detail or argv[0]}"
+            )
+        return completed.stdout
+
+    def _ssh(self, remote_command: str) -> str:
+        return self._run(
+            [
+                "ssh",
+                "-o",
+                "BatchMode=yes",
+                "-o",
+                f"ConnectTimeout={min(self.context.timeout, 30)}",
+                self._alias(),
+                remote_command,
+            ]
+        )
+
+    def op_clusters(self, options: Mapping[str, Any]) -> Any:
+        del options
+        return list_profiles(self.context.repo_root)
+
+    def op_queues(self, options: Mapping[str, Any]) -> Any:
+        del options
+        output = self._ssh(
+            "LC_ALL=C sinfo -h -o '%P|%a|%l|%D|%C|%G'"
+        )
+        fields = [
+            "partition",
+            "availability",
+            "max_walltime",
+            "nodes",
+            "cpu_state",
+            "gres",
+        ]
+        rows = [_split_pipe(line, fields) for line in output.splitlines() if line.strip()]
+        for row in rows:
+            row["partition"] = row["partition"].rstrip("*")
+        return rows
+
+    def op_limits(self, options: Mapping[str, Any]) -> Any:
+        partition = (
+            options.get("partition")
+            or self.context.profile.get("PARTITION")
+            or self.context.profile.get("PARTITION_CPU")
+        )
+        if not partition:
+            raise BackendError("limits requires --partition or a PARTITION profile field")
+        if not str(partition).replace("-", "").replace("_", "").isalnum():
+            raise BackendError("partition contains unsafe characters")
+        output = self._ssh(
+            f"LC_ALL=C scontrol show partition {shlex.quote(str(partition))} -o"
+        ).strip()
+        parsed: dict[str, str] = {"partition": str(partition)}
+        for token in output.split():
+            if "=" in token:
+                key, value = token.split("=", 1)
+                parsed[key] = value
+        keep = {
+            "partition",
+            "State",
+            "TotalNodes",
+            "TotalCPUs",
+            "DefMemPerCPU",
+            "DefMemPerNode",
+            "MaxMemPerCPU",
+            "MaxMemPerNode",
+            "MaxTime",
+            "MinNodes",
+            "MaxNodes",
+            "AllowAccounts",
+            "AllowQos",
+            "QoS",
+            "TRES",
+        }
+        return {key: value for key, value in parsed.items() if key in keep}
+
+    def op_job(self, options: Mapping[str, Any]) -> Any:
+        job_id = str(require_option(options, "job_id"))
+        if not job_id.replace("_", "").isalnum():
+            raise BackendError("job id contains unsafe characters")
+        fields = [
+            "job_id",
+            "state",
+            "partition",
+            "elapsed",
+            "time_limit",
+            "nodes",
+            "cpus",
+            "reason_or_nodes",
+        ]
+        output = self._ssh(
+            "LC_ALL=C squeue -h "
+            f"-j {shlex.quote(job_id)} -o '%i|%T|%P|%M|%l|%D|%C|%R'"
+        ).strip()
+        if output:
+            return _split_pipe(output.splitlines()[0], fields)
+        output = self._ssh(
+            "LC_ALL=C sacct -n -X "
+            f"-j {shlex.quote(job_id)} "
+            "--format=JobIDRaw,State,Partition,Elapsed,Timelimit,NNodes,NCPUS,ExitCode -P"
+        ).strip()
+        if not output:
+            raise BackendError(f"job {job_id!r} was not found")
+        history_fields = fields[:-1] + ["exit_code"]
+        return _split_pipe(output.splitlines()[0], history_fields)
+
+    def op_logs(self, options: Mapping[str, Any]) -> Any:
+        path = str(require_option(options, "path"))
+        if not path.startswith("/") or "\n" in path or "\x00" in path:
+            raise BackendError("remote log path must be a safe absolute path")
+        lines = int(options.get("lines") or 200)
+        if lines < 1 or lines > 10000:
+            raise BackendError("--lines must be between 1 and 10000")
+        direction = options.get("direction") or "tail"
+        command = "tail" if direction == "tail" else "head"
+        output = self._ssh(
+            f"{command} -n {lines} -- {shlex.quote(path)}"
+        )
+        return {"path": path, "direction": direction, "lines": output.splitlines()}
+
+    def op_submit(self, options: Mapping[str, Any]) -> Any:
+        remote_path = str(require_option(options, "remote_path"))
+        if not remote_path.startswith("/") or "\n" in remote_path:
+            raise BackendError("SSH submit requires a safe absolute --remote-path")
+        output = self._ssh(
+            f"sbatch --parsable -- {shlex.quote(remote_path)}"
+        ).strip()
+        return {"job_id": output.split(";", 1)[0], "raw": output}
+
+    def preview_submit(self, options: Mapping[str, Any]) -> Any:
+        remote_path = str(require_option(options, "remote_path"))
+        if not remote_path.startswith("/") or "\n" in remote_path:
+            raise BackendError("SSH submit requires a safe absolute --remote-path")
+        return {
+            "backend": self.name,
+            "operation": "submit",
+            "command": [
+                "ssh",
+                self._alias(),
+                "sbatch --parsable -- " + shlex.quote(remote_path),
+            ],
+            "remote_path": remote_path,
+        }
+
+    def op_cancel(self, options: Mapping[str, Any]) -> Any:
+        job_id = str(require_option(options, "job_id"))
+        if not job_id.replace("_", "").isalnum():
+            raise BackendError("job id contains unsafe characters")
+        self._ssh(f"scancel -- {shlex.quote(job_id)}")
+        return {"job_id": job_id, "cancelled": True}
+
+    def preview_cancel(self, options: Mapping[str, Any]) -> Any:
+        job_id = str(require_option(options, "job_id"))
+        if not job_id.replace("_", "").isalnum():
+            raise BackendError("job id contains unsafe characters")
+        return {
+            "backend": self.name,
+            "operation": "cancel",
+            "command": ["ssh", self._alias(), "scancel -- " + shlex.quote(job_id)],
+            "job_id": job_id,
+        }
+
+    def op_upload(self, options: Mapping[str, Any]) -> Any:
+        local_path = Path(str(require_option(options, "local_path"))).expanduser()
+        remote_path = str(require_option(options, "remote_path"))
+        if not local_path.is_file():
+            raise BackendError(f"local file does not exist: {local_path}")
+        if not remote_path.startswith("/") or "\n" in remote_path:
+            raise BackendError("remote path must be a safe absolute path")
+        remote_q = shlex.quote(remote_path)
+        name_q = shlex.quote(local_path.name)
+        check = self._ssh(
+            f"target={remote_q}; "
+            f"if test -d \"$target\"; then target=\"$target\"/{name_q}; fi; "
+            "if test -e \"$target\"; then state=EXISTS; else state=MISSING; fi; "
+            "printf '%s\\n%s\\n' \"$state\" \"$target\""
+        ).splitlines()
+        state = check[0] if check else ""
+        destination = check[1] if len(check) > 1 else remote_path
+        if state == "EXISTS" and not options.get("cover"):
+            raise BackendError("remote path already exists; pass --cover to overwrite it")
+        self._run(["scp", str(local_path), f"{self._alias()}:{remote_path}"])
+        return {"local_path": str(local_path), "remote_path": destination}
+
+    def op_download(self, options: Mapping[str, Any]) -> Any:
+        remote_path = str(require_option(options, "remote_path"))
+        local_path = Path(str(require_option(options, "local_path"))).expanduser()
+        if not remote_path.startswith("/") or "\n" in remote_path:
+            raise BackendError("remote path must be a safe absolute path")
+        if local_path.exists() and not options.get("cover"):
+            raise BackendError("local path already exists; pass --cover to overwrite it")
+        self._run(["scp", "-r", f"{self._alias()}:{remote_path}", str(local_path)])
+        return {"remote_path": remote_path, "local_path": str(local_path)}
+
+    def op_exec(self, options: Mapping[str, Any]) -> Any:
+        command = str(require_option(options, "command"))
+        if "\x00" in command:
+            raise BackendError("command contains a NUL byte")
+        output = self._ssh(command)
+        return {"stdout": output}

package/skills/scnet-hpc/scripts/scnet_config.py

@@ -0,0 +1,175 @@
+"""User-local, non-secret configuration for the SCNet CLI."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def config_root() -> Path:
+    base = os.environ.get("XDG_CONFIG_HOME")
+    if base:
+        return Path(base).expanduser() / "scnet-hpc"
+    return Path.home() / ".config" / "scnet-hpc"
+
+
+def config_path() -> Path:
+    return config_root() / "config.json"
+
+
+def ssh_profile_path(cluster: str) -> Path:
+    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", cluster):
+        raise ValueError("unsafe cluster profile name")
+    return config_root() / "ssh" / f"{cluster}.json"
+
+
+def openapi_cache_path() -> Path:
+    return config_root() / "openapi.json"
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (FileNotFoundError, OSError, json.JSONDecodeError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def load_user_config() -> dict[str, Any]:
+    data = _read_json(config_path())
+    ssh = data.setdefault("ssh", {})
+    if isinstance(ssh, dict):
+        clusters = ssh.setdefault("clusters", {})
+        if isinstance(clusters, dict):
+            ssh_dir = config_root() / "ssh"
+            for path in sorted(ssh_dir.glob("*.json")):
+                profile = _read_json(path)
+                if profile:
+                    clusters[path.stem] = profile
+    openapi_cache = _read_json(openapi_cache_path())
+    if openapi_cache:
+        current = data.setdefault("openapi", {})
+        if isinstance(current, dict):
+            current.update(openapi_cache)
+    return data
+
+
+def save_user_config(data: dict[str, Any]) -> Path:
+    path = config_path()
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    path.parent.chmod(0o700)
+    clean = json.loads(json.dumps(data, ensure_ascii=False))
+    cluster = str(clean.get("cluster") or "")
+    ssh = clean.get("ssh")
+    if isinstance(ssh, dict):
+        clusters = ssh.get("clusters")
+        if isinstance(clusters, dict):
+            for name, profile in clusters.items():
+                if isinstance(profile, dict):
+                    save_ssh_profile(str(name), profile)
+        elif cluster and isinstance(ssh.get("username"), str):
+            save_ssh_profile(
+                cluster,
+                {
+                    key: value
+                    for key, value in ssh.items()
+                    if key != "clusters"
+                },
+            )
+    openapi = clean.get("openapi")
+    if isinstance(openapi, dict):
+        _save_json_sidecar(openapi_cache_path(), openapi)
+    clean.pop("ssh", None)
+    clean.pop("openapi", None)
+    fd, temporary = tempfile.mkstemp(
+        prefix=".config.", suffix=".tmp", dir=str(path.parent)
+    )
+    try:
+        os.fchmod(fd, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            json.dump(clean, stream, ensure_ascii=False, indent=2, sort_keys=True)
+            stream.write("\n")
+        os.replace(temporary, path)
+    finally:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+    return path
+
+
+def _save_json_sidecar(path: Path, data: dict[str, Any]) -> Path:
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    path.parent.chmod(0o700)
+    clean = json.loads(json.dumps(data, ensure_ascii=False))
+    fd, temporary = tempfile.mkstemp(
+        prefix=f".{path.stem}.", suffix=".tmp", dir=str(path.parent)
+    )
+    try:
+        os.fchmod(fd, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            json.dump(clean, stream, ensure_ascii=False, indent=2, sort_keys=True)
+            stream.write("\n")
+        os.replace(temporary, path)
+    finally:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+    return path
+
+
+def save_ssh_profile(cluster: str, data: dict[str, Any]) -> Path:
+    return _save_json_sidecar(ssh_profile_path(cluster), data)
+
+
+def reset_openapi_metadata() -> bool:
+    path = openapi_cache_path()
+    try:
+        path.unlink()
+        return True
+    except FileNotFoundError:
+        return False
+
+
+def reset_ssh_metadata(cluster: str) -> bool:
+    path = ssh_profile_path(cluster)
+    try:
+        path.unlink()
+        return True
+    except FileNotFoundError:
+        return False
+
+
+def reset_all_metadata() -> list[Path]:
+    removed: list[Path] = []
+    targets = [config_path(), openapi_cache_path()]
+    targets.extend(sorted((config_root() / "ssh").glob("*.json")))
+    for path in targets:
+        try:
+            path.unlink()
+            removed.append(path)
+        except FileNotFoundError:
+            pass
+    for directory in (config_root() / "ssh", config_root()):
+        try:
+            directory.rmdir()
+        except OSError:
+            pass
+    return removed
+
+
+def redacted_config(data: dict[str, Any]) -> dict[str, Any]:
+    """Return a safe-to-display copy; this config should contain no secrets anyway."""
+    result = json.loads(json.dumps(data, ensure_ascii=False))
+    for section in ("openapi", "ssh"):
+        if isinstance(result.get(section), dict):
+            for key in list(result[section]):
+                lowered = key.lower()
+                if any(marker in lowered for marker in ("key", "token", "secret", "password")):
+                    result[section][key] = "<not displayed>"
+    return result

package/skills/scnet-hpc/scripts/scnet_credentials.py

@@ -0,0 +1,160 @@
+"""Secure OpenAPI credential discovery and storage."""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import shutil
+import subprocess
+from typing import Any
+
+
+SERVICE = "scnet-hpc-openapi"
+
+
+class CredentialError(RuntimeError):
+    """A user-facing credential-store failure."""
+
+
+def _valid(data: Any) -> bool:
+    return isinstance(data, dict) and all(
+        isinstance(data.get(key), str) and data[key]
+        for key in ("user", "access_key", "secret_key")
+    )
+
+
+def environment_credentials() -> dict[str, str] | None:
+    data = {
+        "user": os.environ.get("SCNET_OPENAPI_USER", ""),
+        "access_key": os.environ.get("SCNET_OPENAPI_ACCESS_KEY", ""),
+        "secret_key": os.environ.get("SCNET_OPENAPI_SECRET_KEY", ""),
+    }
+    return data if _valid(data) else None
+
+
+def secure_store_name() -> str | None:
+    if platform.system() == "Darwin" and shutil.which("security"):
+        return "macOS Keychain"
+    if shutil.which("secret-tool"):
+        return "Secret Service"
+    return None
+
+
+def load_secure_credentials() -> dict[str, str] | None:
+    provider = secure_store_name()
+    if provider == "macOS Keychain":
+        command = ["security", "find-generic-password", "-s", SERVICE, "-w"]
+    elif provider == "Secret Service":
+        command = ["secret-tool", "lookup", "service", SERVICE]
+    else:
+        return None
+    completed = subprocess.run(
+        command,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    )
+    if completed.returncode != 0 or not completed.stdout.strip():
+        return None
+    try:
+        data = json.loads(completed.stdout)
+    except json.JSONDecodeError:
+        return None
+    return data if _valid(data) else None
+
+
+def load_openapi_credentials() -> tuple[dict[str, str] | None, str | None]:
+    data = environment_credentials()
+    if data:
+        return data, "environment"
+    data = load_secure_credentials()
+    if data:
+        return data, secure_store_name()
+    return None, None
+
+
+def store_openapi_credentials(
+    user: str, access_key: str, secret_key: str
+) -> str:
+    data = {
+        "user": user,
+        "access_key": access_key,
+        "secret_key": secret_key,
+    }
+    if not _valid(data):
+        raise CredentialError("OpenAPI credentials are incomplete")
+    payload = json.dumps(data, ensure_ascii=False, separators=(",", ":"))
+    provider = secure_store_name()
+    if provider == "macOS Keychain":
+        command = [
+            "security",
+            "add-generic-password",
+            "-U",
+            "-s",
+            SERVICE,
+            "-a",
+            user,
+            "-w",
+            payload,
+        ]
+        completed = subprocess.run(
+            command,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+    elif provider == "Secret Service":
+        command = [
+            "secret-tool",
+            "store",
+            "--label=SCNet OpenAPI",
+            "service",
+            SERVICE,
+            "user",
+            user,
+        ]
+        completed = subprocess.run(
+            command,
+            input=payload,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+    else:
+        raise CredentialError(
+            "no supported secure store; use SCNET_OPENAPI_USER, "
+            "SCNET_OPENAPI_ACCESS_KEY, and SCNET_OPENAPI_SECRET_KEY"
+        )
+    if completed.returncode != 0:
+        raise CredentialError(
+            completed.stderr.strip() or f"failed to write {provider}"
+        )
+    return str(provider)
+
+
+def delete_openapi_credentials() -> tuple[bool, str | None]:
+    provider = secure_store_name()
+    if provider == "macOS Keychain":
+        command = ["security", "delete-generic-password", "-s", SERVICE]
+    elif provider == "Secret Service":
+        command = ["secret-tool", "clear", "service", SERVICE]
+    else:
+        return False, None
+    completed = subprocess.run(
+        command,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+        check=False,
+    )
+    if completed.returncode == 0:
+        return True, provider
+    if load_secure_credentials() is None:
+        return False, provider
+    raise CredentialError(
+        completed.stderr.strip() or f"failed to clear {provider}"
+    )

package/skills/scnet-hpc/scripts/scnet_sdk/__init__.py

@@ -0,0 +1,11 @@
+"""Small internal client used by CLI, plugins, and future MCP adapters."""
+
+from .client import SCNetClient, canonical_signature, service_endpoint
+from .notebook import NotebookService
+
+__all__ = [
+    "NotebookService",
+    "SCNetClient",
+    "canonical_signature",
+    "service_endpoint",
+]