dsh-crate-web 0.1.1 → 0.1.2-preview.2

package/core/dsh_pack/preflight.py

@@ -12,6 +12,7 @@ from typing import Any, Mapping
 
 from .errors import DshPackError, IntegrityError, SchemaValidationError, _default_checks
 from .network import network_source_details
+from .identity import expected_install_version, unknown_source
 from .pack import _read_entries, sha256_bytes
 from .validation import validate_manifest, validate_plugins_lock
 
@@ -47,7 +48,7 @@ _EXPECTED_BY_CODE = {
     "HASH_MISMATCH": "declared SHA-256 matches artifact bytes",
     "INTEGRITY_FILE_MISSING": "all declared Pack files are present",
     "INTEGRITY_FILE_SET_MISMATCH": "Pack files exactly match the integrity set",
-    "VERSION_MISSING": "each plugin has resolved.version",
+    "VERSION_MISSING": "each plugin has a known resolution or an explicit unknown resolution with known runtime identity",
     "PACKAGE_IDENTITY_DRIFT": "artifact package identity matches lock identity",
     "REQUIRED_ARTIFACT_MISSING": "required embedded artifact is present",
     "OPTIONAL_ARTIFACT_MISSING": "optional embedded artifact is present",
@@ -70,7 +71,7 @@ _NEXT_STEP_BY_CODE = {
     "HASH_MISMATCH": "Restore the original artifact or regenerate the Pack.",
     "INTEGRITY_FILE_MISSING": "Restore the missing Pack member and regenerate its integrity table.",
     "INTEGRITY_FILE_SET_MISMATCH": "Regenerate the Pack so its integrity file set matches its contents.",
-    "VERSION_MISSING": "Regenerate plugins.lock.json with a resolved plugin version.",
+    "VERSION_MISSING": "Re-export runtime identity, or provide recorded resolution evidence without guessing it.",
     "PACKAGE_IDENTITY_DRIFT": "Restore the artifact matching plugins.lock.json or re-export the Profile.",
     "REQUIRED_ARTIFACT_MISSING": "Restore the required embedded artifact before continuing.",
     "OPTIONAL_ARTIFACT_MISSING": "Restore the optional artifact or intentionally export the plugin reference-only.",
@@ -108,7 +109,9 @@ class Finding:
             raise ValueError(f"unsupported preflight severity: {self.severity}")
 
     def as_dict(self) -> dict[str, Any]:
+        from .diagnostics import envelope_fields
         return {
+            **envelope_fields(operation="inspect", status="FAIL" if self.severity == "BLOCKER" else "UNTESTED"),
             "severity": self.severity,
             "code": self.code,
             "message": self.message,
@@ -138,6 +141,7 @@ class PreflightContext:
     available_secrets: set[str] | None = None
     available_plugins: Mapping[str, str | None] | None = None
     allow_network_reference_install: bool = False
+    verified_local_plugins: Mapping[str, str] = field(default_factory=dict)
 
 
 @dataclass
@@ -384,7 +388,9 @@ def inspect_pack(
 
     try:
         validate_manifest(manifest, require_integrity=True)
-    except SchemaValidationError as error:
+        from .portability import validate_inventory
+        validate_inventory(manifest, {name[len("user-data/"):]: content for name, content in entries.items() if name.startswith("user-data/")}, lock)
+    except DshPackError as error:
         findings.append(_finding("BLOCKER", "SCHEMA_ERROR", str(error)))
 
     raw_plugins = lock.get("plugins", [])
@@ -400,15 +406,14 @@ def inspect_pack(
             if name in plugin_names:
                 findings.append(_finding("BLOCKER", "SCHEMA_ERROR", f"duplicate plugin name: {name}"))
             plugin_names.add(name)
-        resolved = raw_plugin.get("resolved")
-        version = resolved.get("version") if isinstance(resolved, Mapping) else None
+        version = expected_install_version(raw_plugin)
         if not isinstance(version, str) or not version.strip():
             missing_version_indexes.add(index)
             findings.append(
                 _finding(
                     "BLOCKER",
                     "VERSION_MISSING",
-                    f"plugin at plugins.lock.plugins[{index}] is missing resolved.version",
+                    f"plugin at plugins.lock.plugins[{index}] has no usable installation version identity",
                     index=index,
                     plugin=name,
                 )
@@ -466,7 +471,9 @@ def inspect_pack(
 
     try:
         validate_plugins_lock(lock, artifact_paths=artifact_entries)
-    except SchemaValidationError as error:
+        from .validation import validate_format_pair
+        validate_format_pair(manifest, lock)
+    except DshPackError as error:
         message = str(error)
         if "resolved.version" not in message and "resolved is missing fields" not in message:
             if "artifact.path is missing from plugins/" not in message:
@@ -594,9 +601,9 @@ def inspect_pack(
             if not _version_matches(required, current):
                 findings.append(
                     _finding(
-                        "WARNING",
+                        "BLOCKER",
                         "ENVIRONMENT_VERSION_MISMATCH",
-                        f"Pack was exported with {label} {required}; current {label} is {current} (runtime compatibility may differ)",
+                        f"Pack requires {label} {required}; current is {current}",
                         stage="environment",
                         item=section,
                         expected=required,
@@ -636,17 +643,35 @@ def inspect_pack(
     for name in missing_secrets:
         findings.append(_finding("WARNING", "SECRET_MISSING", f"required Secret is missing: {name}", secret=name))
 
+    for plugin in plugin_summaries:
+        artifact = plugin.get("artifact")
+        if not isinstance(artifact, Mapping) or artifact.get("mode") != "reference-only" or not unknown_source(plugin):
+            continue
+        expected_version = expected_install_version(plugin)
+        local_version = context.verified_local_plugins.get(plugin["name"])
+        if expected_version is None or local_version != expected_version:
+            findings.append(_finding(
+                "BLOCKER" if plugin.get("required") is True else "WARNING", "SOURCE_UNKNOWN",
+                f"reference-only plugin has no recorded installation source: {plugin['name']}",
+                stage="identity", plugin=plugin["name"], expected=expected_version,
+                observed=local_version, evidence={"requested": plugin.get("requested"),
+                                                "resolved": plugin.get("resolved"), "runtime": plugin.get("runtime")},
+                suggested_next_step="Provide an explicit local package source with matching name/version, or embed its verified artifact.",
+            ))
+
     if context.available_plugins is not None:
         available_plugins = dict(context.available_plugins)
         for plugin in plugin_summaries:
             artifact = plugin.get("artifact")
             if not isinstance(artifact, Mapping) or artifact.get("mode") != "reference-only":
                 continue
+            if unknown_source(plugin):
+                continue  # Checked separately using verified local-source evidence.
             name = plugin["name"]
             if name not in available_plugins:
                 network_source = network_source_details(plugin)
                 if context.allow_network_reference_install and network_source is not None:
-                    expected_version = plugin.get("resolved", {}).get("version") if isinstance(plugin.get("resolved"), Mapping) else None
+                    expected_version = expected_install_version(plugin)
                     findings.append(
                         _finding(
                             "WARNING",
@@ -670,7 +695,7 @@ def inspect_pack(
                 findings.append(_finding(severity, code, f"reference-only plugin is missing: {name}", plugin=name))
                 continue
             installed_version = available_plugins[name]
-            expected_version = plugin.get("resolved", {}).get("version") if isinstance(plugin.get("resolved"), Mapping) else None
+            expected_version = expected_install_version(plugin)
             if installed_version and expected_version and installed_version != expected_version:
                 findings.append(
                     _finding(
@@ -690,12 +715,32 @@ def inspect_pack(
         path = artifact.get("path")
         if not isinstance(path, str) or path not in entries:
             continue
+        bundle = plugin.get("bundle")
+        # Schema 3 adds exhaustive ordered-patch artifact validation. Older
+        # schemas retain their previous Preflight contract; Import still checks
+        # the actual installed patch declarations and contents.
+        if (manifest.get("schemaVersion") == 3
+                and isinstance(bundle, Mapping) and bundle.get("enabled") is True):
+            from .bundle import check_bundle_artifact
+            try:
+                check_bundle_artifact(entries[path], bundle.get("patch"))
+            except DshPackError as error:
+                findings.append(_finding(
+                    "BLOCKER" if plugin.get("required") is True else "WARNING",
+                    "BUNDLE_PATCH_INVALID", f"embedded Bundle patch check failed: {plugin['name']}",
+                    plugin=plugin["name"], error=str(error), patch=bundle.get("patch"),
+                ))
         identity = _artifact_package_identity(entries[path])
+        resolved = plugin.get("resolved")
+        resolution_unknown = isinstance(resolved, Mapping) and resolved.get("version") == "unknown"
         if identity is None:
+            if resolution_unknown:
+                findings.append(_finding("BLOCKER" if plugin.get("required") is True else "WARNING",
+                                         "PACKAGE_IDENTITY_DRIFT", "embedded package identity cannot be verified",
+                                         plugin=plugin["name"]))
             continue
         plugin["artifactIdentity"] = identity
-        resolved = plugin.get("resolved")
-        expected_version = resolved.get("version") if isinstance(resolved, Mapping) else None
+        expected_version = expected_install_version(plugin)
         drift: dict[str, dict[str, str]] = {}
         if identity["name"] != plugin["name"]:
             drift["name"] = {"expected": str(plugin["name"]), "actual": identity["name"]}
@@ -714,7 +759,8 @@ def inspect_pack(
         if drift:
             findings.append(
                 _finding(
-                    "WARNING",
+                    ("BLOCKER" if plugin.get("required") is True else "WARNING")
+                    if resolution_unknown else "WARNING",
                     "PACKAGE_IDENTITY_DRIFT",
                     f"embedded package identity differs from lock identity: {plugin['name']}",
                     plugin=plugin["name"],
@@ -738,6 +784,8 @@ def inspect_pack(
             "path": str(source_path),
             "format": manifest.get("format"),
             "packVersion": manifest.get("packVersion"),
+            "dataInventory": manifest.get("dataInventory", []),
+            "completeness": manifest.get("completeness", {}),
             "profile": manifest.get("profile"),
         },
         schema={"status": schema_status, "version": schema_version},
@@ -778,8 +826,7 @@ def render_text(result: PreflightResult) -> str:
     ]
     if result.plugins:
         for plugin in result.plugins:
-            resolved = plugin.get("resolved")
-            version = resolved.get("version", "missing") if isinstance(resolved, Mapping) else "missing"
+            version = expected_install_version(plugin) or "unknown"
             required = "required" if plugin.get("required") else "optional"
             artifact = plugin.get("artifact")
             mode = artifact.get("mode", "unknown") if isinstance(artifact, Mapping) else "unknown"

package/core/dsh_pack/troubleshooting.py

@@ -0,0 +1,94 @@
+"""Read-only diagnosis using the release's existing diagnostic registry."""
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+from pathlib import Path
+from typing import Any, Sequence
+
+from .diagnostics import CRATE_VERSION, classify, compat_status
+
+
+def _issues(value: Any):
+    if isinstance(value, list):
+        for item in value:
+            yield from _issues(item)
+    elif isinstance(value, dict):
+        if isinstance(value.get("code"), str):
+            yield value
+        # Only traverse report containers, never execute or interpret arbitrary
+        # command strings, package configuration or model instructions.
+        for key in ("result", "error", "errors", "diagnostic", "diagnostics", "findings", "steps"):
+            if key in value:
+                yield from _issues(value[key])
+
+
+def diagnose(report: dict[str, Any]) -> dict[str, Any]:
+    if not isinstance(report, dict):
+        raise ValueError("diagnostic report must be a JSON object")
+    version = report.get("crateVersion")
+    schema = report.get("diagnosticSchemaVersion")
+    if (report.get("producer") != "dsh-crate" or not isinstance(version, str)
+            or type(schema) is not int):
+        compatibility = {"status": "UNSUPPORTED", "canProceed": False,
+                         "reason": "diagnostic envelope is incomplete"}
+    else:
+        compatibility = compat_status(
+            skill_version=CRATE_VERSION, dsh_crate_range=">=0.1.0 <0.1.1",
+            supported_schemas=[1], crate_version=version,
+            diagnostic_schema_version=schema,
+        )
+    issues = []
+    for issue in _issues(report):
+        matched = classify(issue)
+        # A registry entry suggests checks, not a proven root cause. Actual
+        # severity, stage and evidence always remain available unchanged.
+        issues.append({
+            "code": issue["code"], "stage": issue.get("stage"),
+            "severity": issue.get("severity"), "known": matched["known"],
+            "suggestedChecks": matched.get("suggestedChecks", []),
+            "registry": matched, "evidence": copy.deepcopy(issue),
+        })
+    supported = compatibility["canProceed"] and bool(issues) and all(i["known"] for i in issues)
+    return {
+        "operation": "diagnose", "readOnly": True, "writeScope": "none",
+        "compatibility": compatibility, "sourceStatus": report.get("status"),
+        "decision": "REVIEW_EVIDENCE" if supported else "STOP",
+        "repairStatus": "UNTESTED", "issues": issues,
+        "nextStep": (
+            "Check the listed evidence, propose the smallest scoped correction, then rerun Verify."
+            if supported else
+            "Obtain a complete report from the matching release; unknown evidence must not trigger repair."
+        ),
+        "sourceReport": copy.deepcopy(report),
+    }
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="dsh-crate diagnose")
+    parser.add_argument("report", help="complete diagnostic JSON file")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args(argv)
+    try:
+        result = diagnose(json.loads(Path(args.report).read_text(encoding="utf-8")))
+    except (OSError, ValueError) as error:
+        result = {"operation": "diagnose", "readOnly": True, "decision": "STOP",
+                  "repairStatus": "UNTESTED", "error": str(error)}
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print(f"Diagnosis: {result['decision']} (read-only)")
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0 if result["decision"] == "REVIEW_EVIDENCE" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+
+def configure_parser(subparsers):
+    parser = subparsers.add_parser("diagnose", help="read-only diagnosis from a complete report")
+    parser.add_argument("report")
+    parser.add_argument("--json", action="store_true")
+    parser.set_defaults(handler=lambda args: main([args.report] + (["--json"] if args.json else [])))

package/core/dsh_pack/user_cli.py

@@ -114,7 +114,9 @@ def _envelope(
         preflight = result.get("plan", {}).get("preflight", {}) if isinstance(result.get("plan"), dict) else {}
         if isinstance(preflight, dict) and preflight.get("status") in {"READY", "NOT_READY"}:
             status = preflight["status"]
+    from .diagnostics import envelope_fields
     return {
+        **envelope_fields(operation=command, status=status, operation_id=operation_id),
         "operationId": operation_id,
         "command": command,
         "status": status,
@@ -207,6 +209,11 @@ def _export_core_args(args: argparse.Namespace, home: Path) -> list[str]:
     if args.include_installed_bundles:
         argv.append("--include-installed-bundles")
     _append_repeated(argv, "--required-secret", args.required_secret)
+    if args.preset != "custom":
+        _append_option(argv, "--preset", args.preset)
+    _append_option(argv, "--data-selection-json", args.data_selection_json)
+    if args.plan_only:
+        argv.append("--plan-only")
     return argv
 
 
@@ -294,6 +301,10 @@ def _finish_operation(
 
 
 def _run_export(args: argparse.Namespace) -> int:
+    if args.plan_only:
+        code, payload = _invoke_core(_export_core_args(args, _dsh_home(args.dsh_home)))
+        _print(payload, as_json=args.json)
+        return code
     home = _operation_home(args, profile=args.profile)
     exit_code, result = _invoke_core(_export_core_args(args, home))
     return _finish_operation("export", args, home, exit_code, result)
@@ -399,6 +410,9 @@ def _build_parser() -> argparse.ArgumentParser:
     export.add_argument("--embed", action="append", default=[], metavar="PLUGIN")
     export.add_argument("--reference-only", action="append", default=[], metavar="PLUGIN")
     export.add_argument("--include-installed-bundles", action="store_true", help="include direct installed Bundle inventory in the exported Profile composition")
+    export.add_argument("--preset", choices=("share", "freeze", "custom"), default="custom")
+    export.add_argument("--data-selection-json", help="JSON array of explicitly selected data paths, kinds and owners")
+    export.add_argument("--plan-only", action="store_true", help="preview export contents without writing a Crate")
     export.add_argument("--required-secret", action="append", default=[], metavar="NAME")
     export.add_argument("--json", action="store_true", help="emit machine-readable JSON")
     export.add_argument("--dsh-home", help="DSH_HOME used for Profile-name resolution and reports")
@@ -445,6 +459,8 @@ def _build_parser() -> argparse.ArgumentParser:
     delete.add_argument("--json", action="store_true", help="emit machine-readable JSON")
     delete.add_argument("--dsh-home", help="DSH_HOME containing the Profile and operation reports")
     delete.set_defaults(handler=_run_delete_profile)
+    from .troubleshooting import configure_parser
+    configure_parser(subparsers)
     return parser
 
 

package/core/dsh_pack/validation.py

@@ -126,16 +126,23 @@ def validate_manifest(value: Any, *, require_integrity: bool = False) -> dict[st
     manifest = _object(value, "manifest")
     # Report an unsupported schema before checking version-specific required
     # fields, so a future schema cannot be misdiagnosed as a malformed v1 Pack.
-    if "schemaVersion" in manifest and manifest["schemaVersion"] != SCHEMA_VERSION:
+    if "schemaVersion" in manifest and (type(manifest["schemaVersion"]) is not int or manifest["schemaVersion"] not in {1, 2, 3}):
         raise SchemaValidationError(
             f"unsupported manifest.schemaVersion: {manifest['schemaVersion']!r}; expected {SCHEMA_VERSION}"
         )
     allowed = {"schemaVersion", "format", "packVersion", "profile", "environment", "requiredSecrets", "integrity"}
+    if manifest.get("schemaVersion") in {2, 3}:
+        _validate_completeness(manifest.get("completeness"))
+        if not isinstance(manifest.get("dataInventory"), list):
+            raise SchemaValidationError("schema 2 requires dataInventory")
+        allowed.update({"exportMode", "dataInventory", "completeness"})
+        if not isinstance(manifest.get("exportMode"), str) or manifest["exportMode"] not in {"share", "freeze", "custom"}:
+            raise SchemaValidationError("schema 2 requires exportMode")
     required = {"schemaVersion", "format", "packVersion", "profile", "environment"}
     if require_integrity:
         required.add("integrity")
     _strict_keys(manifest, allowed, required, "manifest")
-    if manifest["schemaVersion"] != SCHEMA_VERSION:
+    if manifest["schemaVersion"] not in {1, 2, 3}:
         raise SchemaValidationError(
             f"unsupported manifest.schemaVersion: {manifest['schemaVersion']!r}; expected {SCHEMA_VERSION}"
         )
@@ -162,6 +169,30 @@ def validate_manifest(value: Any, *, require_integrity: bool = False) -> dict[st
     return manifest
 
 
+
+def _validate_completeness(value: Any) -> None:
+    data = _object(value, "manifest.completeness")
+    arrays = {"embedded", "referenceOnly", "secretNameOnly", "excluded", "unsupported"}
+    keys = arrays | {"hostDependencies", "offlineRestorable", "verification", "reason"}
+    _strict_keys(data, keys, keys, "manifest.completeness")
+    for key in arrays:
+        if not isinstance(data[key], list) or any(not isinstance(x, str) or not x for x in data[key]):
+            raise SchemaValidationError(f"completeness.{key} must be an array of names")
+    if not isinstance(data["hostDependencies"], list):
+        raise SchemaValidationError("completeness.hostDependencies must be an array")
+    for item in data["hostDependencies"]:
+        host = _object(item, "host dependency")
+        _strict_keys(host, {"name", "version", "requiredBy"}, {"name", "version"}, "host dependency")
+        for key in host:
+            _string(host[key], "host dependency." + key)
+    # Export declarations are not runtime evidence. YES belongs in a separately
+    # executed verification report, never in this format's export claim.
+    expected = "NO" if data["referenceOnly"] else "UNPROVEN"
+    if data["offlineRestorable"] != expected or data["verification"] != "UNTESTED":
+        raise SchemaValidationError("export completeness cannot claim verified restoration")
+    _string(data["reason"], "completeness.reason")
+
+
 def validate_environment(value: Any) -> dict[str, Any]:
     environment = _object(value, "manifest.environment")
     _strict_keys(environment, {"schemaVersion", "os", "node", "dsh"}, {"schemaVersion", "os", "node", "dsh"}, "manifest.environment")
@@ -201,10 +232,37 @@ def _validate_package_name(value: Any, path: str) -> str:
     return name
 
 
+def bundle_patch_paths(value: Any) -> tuple[str, ...]:
+    """Return safe patch paths in declared composition order, accepting legacy strings."""
+    values = [value] if isinstance(value, str) else value
+    if not isinstance(values, list) or not values:
+        raise SchemaValidationError("bundle.patch must be a string or a non-empty ordered array of strings")
+    paths: list[str] = []
+    for item in values:
+        path = _string(item, "bundle.patch item").replace("\\", "/")
+        if path.startswith("./"):
+            path = path[2:]
+        if ":" in path:
+            raise SchemaValidationError("bundle.patch path cannot contain a drive or alternate stream")
+        validate_archive_path(f"package/{path}", expected_root="package")
+        reserved = {"con", "prn", "aux", "nul", *(f"com{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10))}
+        if any(part.rstrip(" .") != part or part.split(".", 1)[0].casefold() in reserved for part in path.split("/")):
+            raise SchemaValidationError("bundle.patch path contains a non-portable Windows filename")
+        if path.casefold() in {prior.casefold() for prior in paths}:
+            raise SchemaValidationError("bundle.patch contains duplicate paths")
+        paths.append(path)
+    return tuple(paths)
+
+
+def validate_format_pair(manifest: Mapping[str, Any], lock: Mapping[str, Any]) -> None:
+    if lock.get("schemaVersion") == 2 and manifest.get("schemaVersion") != 3:
+        raise SchemaValidationError("ordered Bundle patch arrays require manifest schema 3 and plugins.lock schema 2")
+
+
 def validate_plugins_lock(value: Any, *, artifact_paths: set[str] | None = None) -> dict[str, Any]:
     lock = _object(value, "plugins.lock")
     _strict_keys(lock, {"schemaVersion", "plugins"}, {"schemaVersion", "plugins"}, "plugins.lock")
-    if lock["schemaVersion"] != SCHEMA_VERSION:
+    if type(lock["schemaVersion"]) is not int or lock["schemaVersion"] not in {1, 2}:
         raise SchemaValidationError(
             f"unsupported plugins.lock.schemaVersion: {lock['schemaVersion']!r}; expected {SCHEMA_VERSION}"
         )
@@ -250,7 +308,9 @@ def validate_plugins_lock(value: Any, *, artifact_paths: set[str] | None = None)
         bundle_enabled = _boolean(bundle["enabled"], f"{path}.bundle.enabled")
         if bundle_enabled:
             _integer(bundle["order"], f"{path}.bundle.order", minimum=0)
-            _string(bundle["patch"], f"{path}.bundle.patch")
+            if isinstance(bundle["patch"], list) and lock["schemaVersion"] != 2:
+                raise SchemaValidationError("Bundle patch arrays require plugins.lock schema 2")
+            bundle_patch_paths(bundle["patch"])
         else:
             if bundle["order"] is not None or bundle["patch"] is not None:
                 raise SchemaValidationError(f"{path}.bundle.order and patch must be null when bundle.enabled is false")