@aws/ml-container-creator 1.0.2 → 1.0.4

package/templates/do/.eval_helper.py

@@ -0,0 +1,409 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Model Quality Evaluation Helper.
+
+Subcommands:
+    evaluate   - Run evaluation against deployed endpoint, compute metrics
+    eval-write - Write evaluation results to S3/Athena (Parquet)
+
+All output is JSON on stdout for bash consumption.
+"""
+
+import argparse
+import json
+import math
+import os
+import sys
+import time
+
+
+# ── Utility functions ─────────────────────────────────────────────────────────
+
+def _error_exit(message):
+    """Print JSON error to stdout and exit."""
+    print(json.dumps({"error": True, "message": message}))
+    sys.exit(1)
+
+
+def _output(data):
+    """Print JSON result to stdout."""
+    print(json.dumps(data))
+    sys.exit(0)
+
+
+# ── Endpoint invocation ───────────────────────────────────────────────────────
+
+def _invoke_endpoint(endpoint_name, ic_name, region, payload):
+    """Invoke SageMaker endpoint via boto3 runtime.
+
+    Uses InvokeEndpoint with InferenceComponentName header for IC routing.
+    Payload should be an OpenAI-compatible chat completion request.
+
+    Returns: parsed JSON response dict
+    """
+    import boto3
+
+    client = boto3.client('sagemaker-runtime', region_name=region)
+
+    kwargs = {
+        'EndpointName': endpoint_name,
+        'ContentType': 'application/json',
+        'Body': json.dumps(payload),
+    }
+    if ic_name:
+        kwargs['InferenceComponentName'] = ic_name
+
+    try:
+        response = client.invoke_endpoint(**kwargs)
+        body = response['Body'].read().decode('utf-8')
+        return json.loads(body)
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def _score_text(endpoint_name, ic_name, region, prompt, completion):
+    """Score a completion by getting its logprobs via the endpoint.
+
+    Sends prompt + completion and requests logprobs for the completion tokens.
+    Returns sum of token logprobs, or None if logprobs unavailable.
+    """
+    messages = [
+        {"role": "user", "content": prompt},
+        {"role": "assistant", "content": completion},
+    ]
+
+    payload = {
+        "messages": messages,
+        "max_tokens": 1,
+        "temperature": 0.0,
+        "logprobs": True,
+        "top_logprobs": 1,
+    }
+
+    response = _invoke_endpoint(endpoint_name, ic_name, region, payload)
+
+    if "error" in response:
+        return None
+
+    # Extract logprobs from response
+    try:
+        choices = response.get("choices", [])
+        if not choices:
+            return None
+
+        # For scoring, we need the logprobs of the completion tokens
+        # The response format varies — try OpenAI-compatible format
+        logprobs_data = choices[0].get("logprobs")
+        if logprobs_data and "content" in logprobs_data:
+            token_logprobs = [t.get("logprob", 0.0) for t in logprobs_data["content"]]
+            return sum(token_logprobs) if token_logprobs else None
+
+        return None
+    except (KeyError, TypeError, IndexError):
+        return None
+
+
+def _generate_response(endpoint_name, ic_name, region, prompt, max_tokens=256):
+    """Generate a response from the endpoint for generation-based metrics.
+
+    Returns: generated text string, or None on failure.
+    """
+    payload = {
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": max_tokens,
+        "temperature": 0.0,
+    }
+
+    response = _invoke_endpoint(endpoint_name, ic_name, region, payload)
+
+    if "error" in response:
+        return None
+
+    try:
+        choices = response.get("choices", [])
+        if choices:
+            return choices[0].get("message", {}).get("content", "")
+        return None
+    except (KeyError, TypeError, IndexError):
+        return None
+
+
+# ── Metric computation ────────────────────────────────────────────────────────
+
+def _compute_sft_metrics(endpoint_name, ic_name, region, dataset, samples):
+    """Compute SFT evaluation metrics.
+
+    Metrics: perplexity (via logprobs), avg_response_length, format_compliance, exact_match
+    """
+    metrics = {}
+    logprob_scores = []
+    response_lengths = []
+    exact_matches = 0
+    total = 0
+
+    for i, record in enumerate(dataset):
+        if samples and i >= samples:
+            break
+
+        prompt = record.get("prompt", "")
+        reference = record.get("reference", "")
+
+        if not prompt:
+            continue
+
+        total += 1
+
+        # Score via logprobs (for perplexity)
+        if reference:
+            score = _score_text(endpoint_name, ic_name, region, prompt, reference)
+            if score is not None:
+                # Approximate per-token logprob
+                # score is sum of logprobs; we need per-token average
+                # Estimate token count from character length (rough: 4 chars/token)
+                est_tokens = max(1, len(reference) // 4)
+                logprob_scores.append(score / est_tokens)
+
+        # Generate response (for length and exact match)
+        generated = _generate_response(endpoint_name, ic_name, region, prompt)
+        if generated is not None:
+            response_lengths.append(len(generated.split()))
+            if reference and generated.strip() == reference.strip():
+                exact_matches += 1
+
+    # Compute aggregate metrics
+    if logprob_scores:
+        avg_logprob = sum(logprob_scores) / len(logprob_scores)
+        metrics["perplexity"] = round(math.exp(-avg_logprob), 4)
+
+    if response_lengths:
+        metrics["avg_response_length"] = round(sum(response_lengths) / len(response_lengths), 1)
+
+    if total > 0:
+        metrics["exact_match_accuracy"] = round(exact_matches / total, 4)
+
+    metrics["samples_scored"] = total
+
+    return metrics
+
+
+def _compute_dpo_metrics(endpoint_name, ic_name, region, dataset, samples):
+    """Compute DPO evaluation metrics.
+
+    Metrics: reward_accuracy, avg_chosen_logprob, avg_rejected_logprob, reward_margin
+    """
+    metrics = {}
+    chosen_scores = []
+    rejected_scores = []
+    reward_correct = 0
+    total = 0
+
+    for i, record in enumerate(dataset):
+        if samples and i >= samples:
+            break
+
+        prompt = record.get("prompt", "")
+        chosen = record.get("chosen", "")
+        rejected = record.get("rejected", "")
+
+        if not prompt or not chosen or not rejected:
+            continue
+
+        total += 1
+
+        # Score chosen
+        chosen_score = _score_text(endpoint_name, ic_name, region, prompt, chosen)
+        # Score rejected
+        rejected_score = _score_text(endpoint_name, ic_name, region, prompt, rejected)
+
+        if chosen_score is not None and rejected_score is not None:
+            chosen_scores.append(chosen_score)
+            rejected_scores.append(rejected_score)
+            if chosen_score > rejected_score:
+                reward_correct += 1
+
+    # Compute aggregate metrics
+    scored = len(chosen_scores)
+    if scored > 0:
+        metrics["reward_accuracy"] = round(reward_correct / scored, 4)
+        metrics["avg_chosen_logprob"] = round(sum(chosen_scores) / scored, 4)
+        metrics["avg_rejected_logprob"] = round(sum(rejected_scores) / scored, 4)
+        metrics["reward_margin"] = round(
+            (sum(chosen_scores) - sum(rejected_scores)) / scored, 4
+        )
+
+    metrics["pairs_scored"] = scored
+    metrics["samples_evaluated"] = total
+
+    return metrics
+
+
+# ── Dataset loading ───────────────────────────────────────────────────────────
+
+def _load_eval_dataset(eval_dataset_path):
+    """Load evaluation dataset from local JSONL file or S3.
+
+    For this MVP, expects a local JSONL file path.
+    S3 and HF resolution is handled by the bash wrapper.
+
+    Returns: list of dicts
+    """
+    records = []
+
+    if not eval_dataset_path:
+        _error_exit("No evaluation dataset specified. Use --eval-dataset <path>")
+
+    # Handle S3 paths by downloading
+    if eval_dataset_path.startswith("s3://"):
+        import boto3
+        import tempfile
+        s3 = boto3.client('s3')
+        bucket = eval_dataset_path.split('/')[2]
+        key = '/'.join(eval_dataset_path.split('/')[3:])
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.jsonl')
+        s3.download_file(bucket, key, tmp.name)
+        eval_dataset_path = tmp.name
+
+    # Load JSONL
+    try:
+        with open(eval_dataset_path, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    records.append(json.loads(line))
+    except (IOError, json.JSONDecodeError) as e:
+        _error_exit(f"Failed to load eval dataset: {e}")
+
+    if not records:
+        _error_exit("Evaluation dataset is empty")
+
+    return records
+
+
+# ── cmd_evaluate ──────────────────────────────────────────────────────────────
+
+def cmd_evaluate(args):
+    """Run evaluation against deployed endpoint.
+
+    Returns JSON with metrics and metadata.
+    """
+    endpoint_name = args.endpoint_name
+    ic_name = args.ic_name
+    region = args.region or os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')
+    technique = args.technique or ''
+    samples = int(args.samples) if args.samples else None
+
+    # Load eval dataset
+    dataset = _load_eval_dataset(args.eval_dataset)
+
+    # Determine technique and compute metrics
+    if technique.lower() == 'dpo':
+        metrics = _compute_dpo_metrics(endpoint_name, ic_name, region, dataset, samples)
+    else:
+        # Default to SFT metrics (works for any technique)
+        metrics = _compute_sft_metrics(endpoint_name, ic_name, region, dataset, samples)
+
+    # Build result
+    result = {
+        "adapter_name": args.ic_name,
+        "technique": technique or "sft",
+        "model": os.environ.get("MODEL_NAME", ""),
+        "eval_dataset": args.eval_dataset or "",
+        "samples_evaluated": metrics.get("samples_evaluated", metrics.get("samples_scored", 0)),
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "metrics": metrics,
+    }
+
+    _output(result)
+
+
+# ── cmd_eval_write ────────────────────────────────────────────────────────────
+
+def cmd_eval_write(args):
+    """Write evaluation results to S3 as Parquet for Athena.
+
+    Reads a results JSON file and converts to Parquet format.
+    """
+    results_file = args.results_file
+    bucket = args.bucket
+    region = args.region or os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')
+
+    # Read results
+    try:
+        with open(results_file, 'r') as f:
+            data = json.load(f)
+    except (IOError, json.JSONDecodeError) as e:
+        _error_exit(f"Failed to read results file: {e}")
+
+    adapter_name = data.get("adapter_name", "unknown")
+    technique = data.get("technique", "unknown")
+    model = data.get("model", "unknown")
+    timestamp = data.get("timestamp", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
+
+    # Build Parquet record
+    record = {
+        "project_name": os.environ.get("PROJECT_NAME", ""),
+        "model_name": model,
+        "adapter_name": adapter_name,
+        "technique": technique,
+        "eval_dataset": data.get("eval_dataset", ""),
+        "samples_evaluated": data.get("samples_evaluated", 0),
+        "metrics": json.dumps(data.get("metrics", {})),
+        "timestamp": timestamp,
+        "region": region,
+    }
+
+    # Write as JSON lines (Athena can read JSON as well as Parquet)
+    # For MVP, write as JSON lines to S3. Parquet requires pyarrow dep.
+    s3_key = f"evaluations/model={model}/adapter={adapter_name}/{timestamp.replace(':', '-')}.json"
+    s3_uri = f"s3://{bucket}/{s3_key}"
+
+    try:
+        import boto3
+        s3 = boto3.client('s3', region_name=region)
+        s3.put_object(
+            Bucket=bucket,
+            Key=s3_key,
+            Body=json.dumps(record),
+            ContentType='application/json',
+        )
+        _output({"written": True, "s3_uri": s3_uri})
+    except Exception as e:
+        _error_exit(f"Failed to write to S3: {e}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description='Model Quality Evaluation Helper')
+    subparsers = parser.add_subparsers(dest='command', required=True)
+
+    # evaluate
+    eval_parser = subparsers.add_parser('evaluate', help='Run evaluation')
+    eval_parser.add_argument('--endpoint-name', required=True)
+    eval_parser.add_argument('--ic-name', required=True)
+    eval_parser.add_argument('--region')
+    eval_parser.add_argument('--technique', default='')
+    eval_parser.add_argument('--eval-dataset', default='')
+    eval_parser.add_argument('--samples', default='')
+    eval_parser.add_argument('--metrics', default='')
+
+    # eval-write
+    write_parser = subparsers.add_parser('eval-write', help='Write results to S3')
+    write_parser.add_argument('--results-file', required=True)
+    write_parser.add_argument('--bucket', required=True)
+    write_parser.add_argument('--region')
+
+    args = parser.parse_args()
+
+    if args.command == 'evaluate':
+        cmd_evaluate(args)
+    elif args.command == 'eval-write':
+        cmd_eval_write(args)
+    else:
+        _error_exit(f"Unknown command: {args.command}")
+
+
+if __name__ == '__main__':
+    main()

package/templates/do/.register_helper.py

@@ -112,6 +112,74 @@ def _truncate_metadata(props):
     return result
 
 
+def _inject_eval_metrics(metadata, args):
+    """Inject evaluation metrics from .mlcc/eval-results/ into metadata.
+
+    Looks for eval results matching the adapter name or project.
+    Adds metrics with 'eval_' prefix (G4 AC-3.1, AC-3.2).
+    Non-fatal: if no eval results exist, returns metadata unchanged.
+
+    Args:
+        metadata: existing metadata dict (may be None)
+        args: parsed args with project_name, adapter name hints
+
+    Returns:
+        metadata dict with eval metrics injected (or unchanged)
+    """
+    if metadata is None:
+        metadata = {}
+
+    # Determine eval results directory (relative to script location)
+    # Convention: .mlcc/eval-results/<adapter-or-ic-name>.json
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    eval_results_dir = os.path.join(script_dir, "..", ".mlcc", "eval-results")
+
+    if not os.path.isdir(eval_results_dir):
+        return metadata
+
+    # Try to find eval results for this adapter
+    # Prioritize: adapter name from args > any available result
+    adapter_name = getattr(args, 'adapter_name', '') or ''
+
+    # Search for matching eval result file
+    eval_file = None
+    if adapter_name:
+        candidate = os.path.join(eval_results_dir, f"{adapter_name}.json")
+        if os.path.isfile(candidate):
+            eval_file = candidate
+
+    # If no specific adapter match, try to find any recent result
+    if not eval_file:
+        try:
+            json_files = [f for f in os.listdir(eval_results_dir) if f.endswith('.json')]
+            if json_files:
+                # Use most recently modified
+                json_files.sort(key=lambda f: os.path.getmtime(os.path.join(eval_results_dir, f)), reverse=True)
+                eval_file = os.path.join(eval_results_dir, json_files[0])
+        except OSError:
+            pass
+
+    if not eval_file:
+        return metadata
+
+    # Load and inject metrics
+    try:
+        with open(eval_file, 'r') as f:
+            eval_data = json.load(f)
+        metrics = eval_data.get("metrics", {})
+        for metric_name, metric_value in metrics.items():
+            # Add with eval_ prefix, truncate to 256 chars
+            key = f"eval_{metric_name}"
+            str_val = str(metric_value)[:MAX_METADATA_VALUE_LEN]
+            metadata[key] = str_val
+        if metrics:
+            _warn(f"Injected {len(metrics)} eval metric(s) from {os.path.basename(eval_file)}")
+    except (IOError, json.JSONDecodeError, KeyError):
+        pass  # Non-fatal — skip eval metrics if file is unreadable
+
+    return metadata
+
+
 def _build_metadata(args):
     """Build customer_metadata_properties dict from CLI args.
 
@@ -283,7 +351,7 @@ def cmd_register_model(args):
 
     # Step 3: Build inference specification
     container_image = args.container_image or ""
-    model_data_url = args.model_data_url or ""
+    model_data_url = (args.model_data_url or "").rstrip("/")
 
     # Step 4: Create Model Package version (AC-1.2, AC-1.7)
     description = f"{args.deployment_config or 'model'} on {args.instance_type or 'unknown'}"
@@ -437,7 +505,7 @@ def cmd_register_adapter(args):
 
     # Step 3: Build inference specification
     container_image = args.container_image or ""
-    model_data_url = args.model_data_url or ""
+    model_data_url = (args.model_data_url or "").rstrip("/")
 
     # Step 4: Create adapter Model Package version (AC-2.1)
     technique = args.tune_technique or "unknown"
@@ -463,12 +531,21 @@ def cmd_register_adapter(args):
                 "SupportedContentTypes": ["application/json"],
                 "SupportedResponseMIMETypes": ["application/json"],
             }
-            if model_data_url:
+            # ModelDataUrl in InferenceSpecification requires a tar.gz object —
+            # uncompressed S3 prefixes (adapter directories) are not supported.
+            # Store uncompressed paths in metadata instead.
+            if model_data_url and model_data_url.endswith(".tar.gz"):
                 create_params["InferenceSpecification"]["Containers"][0]["ModelDataUrl"] = model_data_url
-        elif model_data_url:
+
+        # Always store model/adapter data URL in metadata for registry queries
+        if model_data_url:
             if not metadata:
                 metadata = {}
             metadata["modelDataUrl"] = model_data_url[:1024]
+
+        # Inject evaluation metrics if available (G4 AC-3.1, AC-3.2)
+        metadata = _inject_eval_metrics(metadata, args)
+
         if metadata:
             create_params["CustomerMetadataProperties"] = metadata
 
@@ -1366,9 +1443,24 @@ def cmd_get_version(args):
     os.environ.setdefault("AWS_REGION", region)
 
     try:
-        from sagemaker.core.resources import ModelPackage
+        import boto3
+        sm_client = boto3.client("sagemaker", region_name=region)
 
-        pkg = ModelPackage.get(model_package_arn=version_arn)
+        # Use boto3 directly — sagemaker-core v2.14 ModelPackage.get() requires
+        # model_package_name (not ARN) and rejects model_package_arn as unexpected kwarg.
+        pkg_response = sm_client.describe_model_package(ModelPackageName=version_arn)
+
+        # Wrap in a simple namespace for consistent access below
+        class _Pkg:
+            def __init__(self, data):
+                self._data = data
+                self.model_package_arn = data.get("ModelPackageArn", version_arn)
+                self.inference_specification = data.get("InferenceSpecification")
+                self.customer_metadata_properties = data.get("CustomerMetadataProperties", {})
+                self.model_approval_status = data.get("ModelApprovalStatus", "")
+                self.model_package_description = data.get("ModelPackageDescription", "")
+                self.creation_time = data.get("CreationTime")
+        pkg = _Pkg(pkg_response)
 
         # Extract model data URL from inference spec
         model_data_url = ""
@@ -1381,6 +1473,10 @@ def cmd_get_version(args):
         # Get metadata
         metadata = getattr(pkg, "customer_metadata_properties", None) or {}
 
+        # Fallback: modelDataUrl stored in metadata when adapter is uncompressed S3 prefix
+        if not model_data_url and metadata.get("modelDataUrl"):
+            model_data_url = metadata["modelDataUrl"]
+
         # Get status
         status = getattr(pkg, "model_approval_status", "") or ""
 
@@ -1414,6 +1510,7 @@ def cmd_resolve_dataset(args):
 
     Version resolution (AC-2.1, AC-2.4):
     - --version N: resolve the Nth version (ordinal, 1-based) for this name
+    - --version X.Y.Z: resolve by semver string match
     - No --version: resolve latest (existing behavior)
     - If requested version doesn't exist: print available versions and exit 1 (AC-2.5)
 
@@ -1421,14 +1518,20 @@ def cmd_resolve_dataset(args):
     or error if not found.
     """
     name = args.name
-    version_ordinal = getattr(args, "version", None)
+    version_spec = getattr(args, "version", None)
 
     if not name:
         _error_exit("--name is required", code="MISSING_ARGUMENT")
 
     # If version is specified, use version-aware resolution
-    if version_ordinal is not None:
-        return _resolve_dataset_version(name, version_ordinal)
+    if version_spec is not None:
+        # Determine if it's an ordinal (pure integer) or semver string
+        try:
+            version_ordinal = int(version_spec)
+            return _resolve_dataset_version(name, version_ordinal)
+        except ValueError:
+            # Not an integer — treat as semver string
+            return _resolve_dataset_version_by_semver(name, version_spec)
 
     # No version — resolve latest (existing behavior)
     # Try SageMaker AI Registry API first
@@ -1545,6 +1648,77 @@ def _resolve_dataset_version(name, version_ordinal):
     _error_exit(f"Dataset not found: {name}", code="DATASET_NOT_FOUND")
 
 
+def _resolve_dataset_version_by_semver(name, version_str):
+    """Resolve a specific version of a named dataset by semver string match.
+
+    Searches the versions[] array for an entry whose 'version' field matches
+    the provided semver string (e.g., '1.0.0').
+
+    If the version doesn't exist, prints available versions and exits 1.
+
+    Args:
+        name: Dataset name
+        version_str: Semver string to match (e.g., '1.0.0', '2.1.0')
+    """
+    # Load local registry
+    entries = _load_registry(_DATASETS_REGISTRY)
+
+    for entry in entries:
+        if entry.get("name") == name:
+            versions = entry.get("versions", [])
+
+            if not versions:
+                # Legacy entry without versions array — treat as having version "1.0.0"
+                if version_str == "1.0.0":
+                    output = dict(entry)
+                    output["version"] = "1.0.0"
+                    output["ordinal"] = 1
+                    if "arn" not in output:
+                        output["arn"] = None
+                    _output(output)
+                else:
+                    print(f"Error: Version {version_str} not found for dataset '{name}'", file=sys.stderr)
+                    print(f"Available versions: 1.0.0", file=sys.stderr)
+                    print(json.dumps({
+                        "error": f"Version {version_str} not found for dataset '{name}'",
+                        "code": "VERSION_NOT_FOUND",
+                        "available_versions": [{"ordinal": 1, "version": "1.0.0"}],
+                    }))
+                    sys.exit(1)
+
+            # Search for matching version string
+            for i, v in enumerate(versions, 1):
+                ver = v.get("version", "")
+                if ver == version_str:
+                    _output({
+                        "name": name,
+                        "s3_uri": v.get("s3_uri", entry.get("s3_uri", "")),
+                        "arn": entry.get("arn"),
+                        "format": v.get("format", entry.get("format", "jsonl")),
+                        "technique": v.get("technique", entry.get("technique", "")),
+                        "version": ver,
+                        "ordinal": i,
+                        "hash": v.get("hash"),
+                    })
+
+            # Version string not found — show available
+            print(f"Error: Version {version_str} not found for dataset '{name}'", file=sys.stderr)
+            available = []
+            for i, v in enumerate(versions, 1):
+                ver = v.get("version", f"{i}.0.0")
+                available.append({"ordinal": i, "version": ver})
+                print(f"  v{i} ({ver})", file=sys.stderr)
+            print(json.dumps({
+                "error": f"Version {version_str} not found for dataset '{name}'",
+                "code": "VERSION_NOT_FOUND",
+                "available_versions": available,
+            }))
+            sys.exit(1)
+
+    # Dataset name not found at all
+    _error_exit(f"Dataset not found: {name}", code="DATASET_NOT_FOUND")
+
+
 # ── Subcommand: resolve-evaluator ────────────────────────────────────────────
 
 
@@ -1706,8 +1880,8 @@ def main():
         help="Resolve a registered dataset by name",
     )
     resolve_dataset_parser.add_argument("--name", required=True, help="Dataset name to resolve")
-    resolve_dataset_parser.add_argument("--version", type=int, default=None,
-                                        help="Version ordinal to resolve (e.g., 2 for the 2nd version). Default: latest.")
+    resolve_dataset_parser.add_argument("--version", type=str, default=None,
+                                        help="Version to resolve: ordinal (e.g., 2) or semver (e.g., 1.0.0). Default: latest.")
 
     # ── resolve-evaluator ─────────────────────────────────────────────────
     resolve_evaluator_parser = subparsers.add_parser(