@groupby/ai-dev 0.5.7 → 0.5.9

package/teams/fhr-ai-team/skills/ml-tooling-dev/scripts/mlflow_query.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""
+Query the dev MLflow tracking server for runs, metrics, and registered models.
+
+Usage:
+    python3 mlflow_query.py experiments                              # List all experiments
+    python3 mlflow_query.py runs <experiment_name>                   # List runs in experiment
+    python3 mlflow_query.py run <run_id>                             # Show run details + metrics
+    python3 mlflow_query.py models                                   # List registered models
+    python3 mlflow_query.py model <model_name>                       # Show model versions + aliases
+    python3 mlflow_query.py model-for-predictor <predictor_id>       # Find model by predictor_id
+"""
+
+import argparse
+import json
+import re
+import sys
+from datetime import datetime
+from urllib.request import urlopen, Request
+from urllib.parse import urlencode, quote
+from urllib.error import URLError
+
+MLFLOW_HOST = "http://10.11.96.16:5000"
+API_BASE = f"{MLFLOW_HOST}/api/2.0/mlflow"
+
+# Only allow alphanumeric, hyphens, underscores, dots, and spaces in filter values
+_SAFE_FILTER_VALUE = re.compile(r"^[\w.\- ]+$")
+
+
+def get(path: str, params: dict = None) -> dict:
+    url = f"{API_BASE}{path}"
+    if params:
+        url = f"{url}?{urlencode(params)}"
+    try:
+        with urlopen(url, timeout=15) as r:
+            return json.loads(r.read())
+    except URLError as e:
+        print(f"[ERROR] Cannot reach MLflow at {url}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def get_with_name(path: str, name: str, extra_params: dict = None) -> dict:
+    """GET with model name properly URL-encoded (avoids double-encoding from urlencode)."""
+    encoded_name = quote(name, safe="")
+    url = f"{API_BASE}{path}?name={encoded_name}"
+    if extra_params:
+        url += "&" + urlencode(extra_params)
+    try:
+        with urlopen(url, timeout=15) as r:
+            return json.loads(r.read())
+    except URLError as e:
+        print(f"[ERROR] Cannot reach MLflow at {url}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def post(path: str, body: dict) -> dict:
+    url = f"{API_BASE}{path}"
+    data = json.dumps(body).encode()
+    req = Request(url, data=data, headers={"Content-Type": "application/json"})
+    try:
+        with urlopen(req, timeout=15) as r:
+            return json.loads(r.read())
+    except URLError as e:
+        print(f"[ERROR] Cannot reach MLflow at {url}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def list_experiments() -> None:
+    data = get("/experiments/search", {"max_results": 100, "order_by": "last_update_time DESC"})
+    experiments = data.get("experiments", [])
+    print(f"{'Experiment ID':<15}  {'Name'}")
+    print("-" * 80)
+    for exp in experiments:
+        print(f"{exp['experiment_id']:<15}  {exp['name']}")
+
+
+def list_runs(experiment_name: str) -> None:
+    data = get("/experiments/search", {"max_results": 200})
+    exp_id = None
+    for exp in data.get("experiments", []):
+        if experiment_name.lower() in exp["name"].lower():
+            exp_id = exp["experiment_id"]
+            print(f"Experiment: {exp['name']} (ID: {exp_id})")
+            break
+    if not exp_id:
+        print(f"[ERROR] No experiment found matching '{experiment_name}'")
+        list_experiments()
+        sys.exit(1)
+
+    runs_data = post("/runs/search", {
+        "experiment_ids": [exp_id],
+        "max_results": 30,
+        "order_by": ["start_time DESC"]
+    })
+    runs = runs_data.get("runs", [])
+
+    print(f"\n{'Run ID':<36}  {'Status':<12}  {'Start':<22}  Name")
+    print("-" * 100)
+    for run in runs:
+        info = run.get("info", {})
+        run_id = info.get("run_id", "")
+        status = info.get("status", "")
+        start = info.get("start_time", 0)
+        run_name = info.get("run_name", "")
+        start_str = datetime.fromtimestamp(start / 1000).strftime("%Y-%m-%d %H:%M") if start else ""
+        print(f"{run_id:<36}  {status:<12}  {start_str:<22}  {run_name}")
+
+
+def show_run(run_id: str) -> None:
+    data = get("/runs/get", {"run_id": run_id})
+    run = data.get("run", {})
+    info = run.get("info", {})
+    params = run.get("data", {}).get("params", [])
+    metrics = run.get("data", {}).get("metrics", [])
+    tags = run.get("data", {}).get("tags", [])
+
+
+    start = info.get("start_time", 0)
+    end = info.get("end_time", 0)
+    start_str = datetime.fromtimestamp(start / 1000).strftime("%Y-%m-%d %H:%M:%S") if start else "N/A"
+    end_str = datetime.fromtimestamp(end / 1000).strftime("%Y-%m-%d %H:%M:%S") if end else "N/A"
+
+    print(f"Run ID:    {info.get('run_id', '')}")
+    print(f"Name:      {info.get('run_name', '')}")
+    print(f"Status:    {info.get('status', '')}")
+    print(f"Started:   {start_str}")
+    print(f"Finished:  {end_str}")
+    print(f"Artifact:  {info.get('artifact_uri', '')}")
+    print()
+
+    if params:
+        print("PARAMS:")
+        for p in sorted(params, key=lambda x: x["key"]):
+            print(f"  {p['key']}: {p['value']}")
+        print()
+
+    if metrics:
+        print("METRICS:")
+        for m in sorted(metrics, key=lambda x: x["key"]):
+            val = m['value']
+            print(f"  {m['key']}: {val:.6f}" if isinstance(val, float) else f"  {m['key']}: {val}")
+        print()
+
+    relevant_tags = {t["key"]: t["value"] for t in tags if not t["key"].startswith("mlflow.log")}
+    if relevant_tags:
+        print("TAGS:")
+        for k, v in sorted(relevant_tags.items()):
+            print(f"  {k}: {v}")
+
+
+def list_models() -> None:
+    data = get("/registered-models/search", {"max_results": 100})
+    models = data.get("registered_models", [])
+    print(f"{'Model Name':<60}  {'Latest Version':<16}  Latest Run ID")
+    print("-" * 110)
+    for m in models:
+        name = m.get("name", "")
+        latest = m.get("latest_versions", [])
+        version = latest[0].get("version", "") if latest else ""
+        run_id = latest[0].get("run_id", "") if latest else ""
+        print(f"{name:<60}  {version:<16}  {run_id}")
+
+
+def show_model(model_name: str) -> None:
+    data = get_with_name("/registered-models/get", model_name)
+    model = data.get("registered_model", {})
+    print(f"Model:  {model.get('name', '')}")
+
+    # Get all versions via search
+    sanitized_name = model_name.replace("'", "''")
+    versions_data = get("/model-versions/search", {
+        "filter": f"name='{sanitized_name}'",
+        "max_results": 20,
+        "order_by": "version_number DESC"
+    })
+    versions = versions_data.get("model_versions", [])
+
+    # Aliases are in the registered_model response
+    aliases = model.get("aliases", [])
+    alias_by_version = {a.get("version"): a.get("alias") for a in aliases}
+
+
+    print(f"\n{'Version':<10}  {'Run ID':<36}  {'Stage':<12}  {'Updated':<22}  Alias")
+    print("-" * 100)
+    for v in versions:
+        version = v.get("version", "")
+        run_id = v.get("run_id", "")
+        stage = v.get("current_stage", "None")
+        ts = v.get("last_updated_timestamp", 0)
+        updated = datetime.fromtimestamp(ts / 1000).strftime("%Y-%m-%d %H:%M") if ts else ""
+        alias = alias_by_version.get(version, "")
+        print(f"{version:<10}  {run_id:<36}  {stage:<12}  {updated:<22}  {alias}")
+
+
+def find_model_for_predictor(predictor_id: str) -> None:
+    if not _SAFE_FILTER_VALUE.match(predictor_id):
+        print(f"[ERROR] Invalid predictor_id: {predictor_id!r}", file=sys.stderr)
+        sys.exit(1)
+    data = get("/registered-models/search", {
+        "filter": f"name LIKE '%{predictor_id}%'",
+        "max_results": 20
+    })
+    models = data.get("registered_models", [])
+    if not models:
+        print(f"No registered models found containing: {predictor_id}")
+        print(f"Browse all models: {MLFLOW_HOST}/#/models")
+        return
+    for m in models:
+        print(f"\n{'='*60}")
+        show_model(m["name"])
+
+
+def main():
+    parser = argparse.ArgumentParser(description=f"Query dev MLflow ({MLFLOW_HOST})")
+    sub = parser.add_subparsers(dest="cmd")
+
+    sub.add_parser("experiments", help="List all experiments")
+
+    p_runs = sub.add_parser("runs", help="List runs in experiment")
+    p_runs.add_argument("experiment_name", help="Experiment name (partial match)")
+
+    p_run = sub.add_parser("run", help="Show run details + metrics")
+    p_run.add_argument("run_id", help="MLflow run ID")
+
+    sub.add_parser("models", help="List registered models")
+
+    p_model = sub.add_parser("model", help="Show model versions and aliases")
+    p_model.add_argument("model_name", help="Registered model name")
+
+    p_pred = sub.add_parser("model-for-predictor", help="Find model by predictor_id")
+    p_pred.add_argument("predictor_id", help="Predictor ID to search for")
+
+    args = parser.parse_args()
+
+    if args.cmd == "experiments":
+        list_experiments()
+    elif args.cmd == "runs":
+        list_runs(args.experiment_name)
+    elif args.cmd == "run":
+        show_run(args.run_id)
+    elif args.cmd == "models":
+        list_models()
+    elif args.cmd == "model":
+        show_model(args.model_name)
+    elif args.cmd == "model-for-predictor":
+        find_model_for_predictor(args.predictor_id)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

package/teams/fhr-ai-team/skills/ml-tooling-dev/scripts/mongo_predictor.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""
+Read, update, apply, or diff predictor training config in dev MongoDB.
+
+All operations target: mongodb://10.11.96.21:27017/earlybirds (collection: predictors)
+
+Usage:
+    mongo_predictor.py read   <predictor_id> [--strategy <name>]
+    mongo_predictor.py update <predictor_id> [--strategy <name>] --set k=v [--set k=v ...]
+    mongo_predictor.py apply  <predictor_id> [--strategy <name>] --file <config.json>
+    mongo_predictor.py diff   <predictor_id> [--strategy <name>] --file <config.json>
+
+Strategy defaults to "semantic-search-learning" (the documented use case).
+The strategy config lives at: config.batch.<strategy>
+
+Examples:
+    # Read current config
+    mongo_predictor.py read 64f0a12b5856b11b7aa4e71e
+
+    # Patch a few fields (dot-notation $set under config.batch.<strategy>)
+    mongo_predictor.py update 64f0a12b5856b11b7aa4e71e \\
+        --set pipelineConfig.maxSequenceLength=64 \\
+        --set learningConfig.trainingArguments.perDeviceTrainBatchSize=128 \\
+        --set learningConfig.trainingArguments.gradientCheckpointing=true
+
+    # Replace the whole strategy config from a file (uses NumberInt for ints)
+    mongo_predictor.py apply 64f0a12b5856b11b7aa4e71e --file experiment-A.json
+
+    # Show what apply would change, before doing it
+    mongo_predictor.py diff 64f0a12b5856b11b7aa4e71e --file experiment-A.json
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+MONGO_URI = "mongodb://10.11.96.21:27017/earlybirds"
+COLLECTION = "predictors"
+DEFAULT_STRATEGY = "semantic-search-learning"
+MONGOSH_TIMEOUT_SECONDS = 30
+
+OBJECTID_RE = re.compile(r"^[0-9a-fA-F]{24}$")
+
+
+def require_mongosh() -> None:
+    if shutil.which("mongosh") is None:
+        print("[ERROR] mongosh not found. Install with: brew install mongosh", file=sys.stderr)
+        sys.exit(1)
+
+
+def validate_object_id(oid: str) -> None:
+    if not OBJECTID_RE.match(oid):
+        print(
+            f"[ERROR] Invalid predictor_id: {oid!r} (expected 24-char hex ObjectId)",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+
+def run_mongosh(js: str, payload: dict) -> dict:
+    """Run a mongosh --eval template. Payload is bound via env var as JSON.
+
+    The js template MUST read its inputs from process.env.PAYLOAD and emit a
+    single JSON object via print(JSON.stringify(...)). User-controlled values
+    never appear in the js text itself, only in the env-bound JSON payload.
+    """
+    require_mongosh()
+    env = os.environ.copy()
+    env["PAYLOAD"] = json.dumps(payload)
+    cmd = ["mongosh", MONGO_URI, "--quiet", "--eval", js]
+    try:
+        result = subprocess.run(
+            cmd, env=env, capture_output=True, text=True, timeout=MONGOSH_TIMEOUT_SECONDS
+        )
+    except subprocess.TimeoutExpired:
+        print(f"[ERROR] mongosh timed out after {MONGOSH_TIMEOUT_SECONDS}s", file=sys.stderr)
+        sys.exit(1)
+    if result.returncode != 0:
+        print(f"[ERROR] mongosh failed (exit {result.returncode}):", file=sys.stderr)
+        if result.stderr.strip():
+            print(result.stderr.strip(), file=sys.stderr)
+        if result.stdout.strip():
+            print(result.stdout.strip(), file=sys.stderr)
+        sys.exit(1)
+    out = result.stdout.strip()
+    if not out:
+        return {}
+    try:
+        return json.loads(out)
+    except json.JSONDecodeError as e:
+        print(f"[ERROR] Could not parse mongosh output as JSON: {e}", file=sys.stderr)
+        print(out, file=sys.stderr)
+        sys.exit(1)
+
+
+def coerce_value(raw: str):
+    """Coerce a CLI string value to a typed Python value for $set."""
+    if raw == "null":
+        return None
+    if raw == "true":
+        return True
+    if raw == "false":
+        return False
+    # JSON literal (object, array, quoted string)
+    if raw and raw[0] in '{["':
+        try:
+            return json.loads(raw)
+        except json.JSONDecodeError:
+            pass
+    # Number
+    try:
+        if "." in raw or "e" in raw.lower():
+            return float(raw)
+        return int(raw)
+    except ValueError:
+        pass
+    return raw
+
+
+def to_extended_json_ints(obj):
+    """Recursively rewrite Python ints as MongoDB extended-JSON NumberInt form.
+
+    Used by `apply` so a full-strategy $set stores integer fields as Int32
+    instead of Double (see mongodb-config.md gotchas).
+    """
+    if isinstance(obj, bool):
+        return obj  # bool is a subclass of int; keep as bool
+    if isinstance(obj, int):
+        return {"$numberInt": str(obj)}
+    if isinstance(obj, dict):
+        return {k: to_extended_json_ints(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [to_extended_json_ints(v) for v in obj]
+    return obj
+
+
+def fetch_strategy_config(predictor_id: str, strategy: str) -> dict:
+    """Return the current config.batch.<strategy> for a predictor, or {} if absent."""
+    payload = {"id": predictor_id, "strategy": strategy}
+    js = """
+    const p = JSON.parse(process.env.PAYLOAD);
+    const doc = db.getCollection("predictors").findOne({_id: ObjectId(p.id)});
+    if (!doc) { print(JSON.stringify({found: false})); quit(0); }
+    const cfg = (doc.config && doc.config.batch && doc.config.batch[p.strategy]) || null;
+    print(JSON.stringify({found: true, config: cfg}));
+    """
+    result = run_mongosh(js, payload)
+    if not result.get("found"):
+        print(f"[ERROR] No predictor matched _id={predictor_id}", file=sys.stderr)
+        sys.exit(2)
+    return result.get("config") or {}
+
+
+def cmd_read(predictor_id: str, strategy: str) -> None:
+    cfg = fetch_strategy_config(predictor_id, strategy)
+    if not cfg:
+        print(
+            f"[WARN] Predictor {predictor_id} has no config.batch.{strategy}",
+            file=sys.stderr,
+        )
+        print("{}")
+        return
+    print(json.dumps(cfg, indent=2, sort_keys=True))
+
+
+def cmd_update(predictor_id: str, strategy: str, sets: list) -> None:
+    if not sets:
+        print("[ERROR] update requires at least one --set k=v", file=sys.stderr)
+        sys.exit(1)
+
+    set_doc = {}
+    for kv in sets:
+        if "=" not in kv:
+            print(f"[ERROR] --set value must be k=v, got: {kv!r}", file=sys.stderr)
+            sys.exit(1)
+        key, _, raw = kv.partition("=")
+        key = key.strip()
+        if not key:
+            print(f"[ERROR] empty key in --set: {kv!r}", file=sys.stderr)
+            sys.exit(1)
+        full_key = f"config.batch.{strategy}.{key}"
+        set_doc[full_key] = coerce_value(raw)
+
+    payload = {"id": predictor_id, "set": set_doc}
+    js = """
+    const p = JSON.parse(process.env.PAYLOAD);
+    const r = db.getCollection("predictors").updateOne(
+        {_id: ObjectId(p.id)},
+        {$set: p.set}
+    );
+    print(JSON.stringify({matched: r.matchedCount, modified: r.modifiedCount}));
+    """
+    result = run_mongosh(js, payload)
+    matched = result.get("matched", 0)
+    modified = result.get("modified", 0)
+    print(f"matched={matched} modified={modified}")
+    if matched == 0:
+        print(f"[ERROR] No predictor matched _id={predictor_id}", file=sys.stderr)
+        sys.exit(2)
+    print("Set fields:")
+    for k, v in set_doc.items():
+        print(f"  {k} = {json.dumps(v)}")
+
+
+def cmd_apply(predictor_id: str, strategy: str, file_path: str) -> None:
+    new_cfg = load_json_file(file_path)
+    if not isinstance(new_cfg, dict):
+        print(f"[ERROR] {file_path} must contain a JSON object at the top level", file=sys.stderr)
+        sys.exit(1)
+
+    full_key = f"config.batch.{strategy}"
+    set_doc = {full_key: to_extended_json_ints(new_cfg)}
+    payload = {"id": predictor_id, "set": set_doc}
+
+    # EJSON.parse honors {"$numberInt": "N"} and stores as Int32. Bare floats
+    # remain Double, which is what we want.
+    js = """
+    const p = EJSON.parse(process.env.PAYLOAD);
+    const r = db.getCollection("predictors").updateOne(
+        {_id: ObjectId(p.id)},
+        {$set: p.set}
+    );
+    print(JSON.stringify({matched: r.matchedCount, modified: r.modifiedCount}));
+    """
+    result = run_mongosh(js, payload)
+    matched = result.get("matched", 0)
+    modified = result.get("modified", 0)
+    print(f"matched={matched} modified={modified}")
+    if matched == 0:
+        print(f"[ERROR] No predictor matched _id={predictor_id}", file=sys.stderr)
+        sys.exit(2)
+    print(f"Applied config.batch.{strategy} from {file_path}")
+
+
+def cmd_diff(predictor_id: str, strategy: str, file_path: str) -> None:
+    new_cfg = load_json_file(file_path)
+    if not isinstance(new_cfg, dict):
+        print(f"[ERROR] {file_path} must contain a JSON object at the top level", file=sys.stderr)
+        sys.exit(1)
+
+    current = fetch_strategy_config(predictor_id, strategy)
+    diffs = compute_diff(current, new_cfg, prefix="")
+    if not diffs:
+        print(f"No differences. Current config.batch.{strategy} matches {file_path}.")
+        return
+    print(f"Diff (current -> {file_path}):")
+    for op, path, before, after in diffs:
+        if op == "+":
+            print(f"  + {path} = {json.dumps(after)}")
+        elif op == "-":
+            print(f"  - {path}  (was {json.dumps(before)})")
+        else:
+            print(f"  ~ {path}: {json.dumps(before)} -> {json.dumps(after)}")
+
+
+def load_json_file(file_path: str):
+    try:
+        with open(file_path) as f:
+            return json.load(f)
+    except FileNotFoundError:
+        print(f"[ERROR] File not found: {file_path}", file=sys.stderr)
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        print(f"[ERROR] Invalid JSON in {file_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+    except OSError as e:
+        print(f"[ERROR] Could not read {file_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def compute_diff(a, b, prefix: str) -> list:
+    """Return [(op, path, before, after)] where op is '+', '-', or '~'."""
+    diffs = []
+    if isinstance(a, dict) and isinstance(b, dict):
+        for k in sorted(set(a) | set(b)):
+            sub = f"{prefix}.{k}" if prefix else k
+            if k not in a:
+                diffs.append(("+", sub, None, b[k]))
+            elif k not in b:
+                diffs.append(("-", sub, a[k], None))
+            else:
+                diffs.extend(compute_diff(a[k], b[k], sub))
+    elif a != b:
+        diffs.append(("~", prefix, a, b))
+    return diffs
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=f"Manage predictor training config in dev MongoDB ({MONGO_URI})",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="See references/mongodb-config.md for the full strategy config schema.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("predictor_id", help="MongoDB ObjectId (24 hex chars)")
+    common.add_argument(
+        "--strategy",
+        default=DEFAULT_STRATEGY,
+        help=f"Strategy ID under config.batch (default: {DEFAULT_STRATEGY})",
+    )
+
+    sub.add_parser("read", parents=[common], help="Print current config.batch.<strategy>")
+
+    p_update = sub.add_parser(
+        "update",
+        parents=[common],
+        help="Patch fields under config.batch.<strategy> via dot-notation $set",
+    )
+    p_update.add_argument(
+        "--set",
+        action="append",
+        dest="sets",
+        default=[],
+        metavar="K=V",
+        help="Field path under strategy + value. Repeat for multiple fields.",
+    )
+
+    p_apply = sub.add_parser(
+        "apply",
+        parents=[common],
+        help="Replace entire config.batch.<strategy> from a JSON file",
+    )
+    p_apply.add_argument("--file", required=True, help="Path to JSON file with new strategy config")
+
+    p_diff = sub.add_parser(
+        "diff",
+        parents=[common],
+        help="Show diff between current config.batch.<strategy> and a JSON file",
+    )
+    p_diff.add_argument("--file", required=True, help="Path to JSON file to compare against")
+
+    args = parser.parse_args()
+    validate_object_id(args.predictor_id)
+
+    if args.cmd == "read":
+        cmd_read(args.predictor_id, args.strategy)
+    elif args.cmd == "update":
+        cmd_update(args.predictor_id, args.strategy, args.sets)
+    elif args.cmd == "apply":
+        cmd_apply(args.predictor_id, args.strategy, args.file)
+    elif args.cmd == "diff":
+        cmd_diff(args.predictor_id, args.strategy, args.file)
+
+
+if __name__ == "__main__":
+    main()