aether-colony 1.1.0

package/.aether/utils/xml-compose.sh

@@ -0,0 +1,253 @@
+#!/bin/bash
+# XML Composition Functions for Worker Priming
+# Part of xml-utils.sh - XInclude-based modular configuration composition
+#
+# Usage: source .aether/utils/xml-compose.sh
+#        xml-compose <input_xml> [output_xml]
+#        xml-compose-worker-priming <priming_xml> [output_xml]
+#        xml-list-includes <xml_file>
+#
+# These functions enable declarative composition of worker configurations
+# using XInclude directives to merge queen-wisdom, active-trails, and stack-profiles.
+
+set -euo pipefail
+
+# Note: This file should be sourced AFTER xml-utils.sh or xml-core.sh
+# It relies on xml_json_ok, xml_json_err, and XMLLINT_AVAILABLE variables
+# Source xml-core.sh for JSON helpers and tool detection if not already loaded
+if ! type xml_json_ok &>/dev/null; then
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    source "$SCRIPT_DIR/xml-core.sh"
+fi
+
+# ============================================================================
+# Path Validation (Security)
+# ============================================================================
+
+# xml-validate-include-path: Validate XInclude path for traversal attacks
+# Usage: xml-validate-include-path <include_path> <base_dir>
+# Returns: absolute path on success, exits with error on failure
+xml-validate-include-path() {
+    local include_path="$1"
+    local base_dir="$2"
+
+    # Resolve base directory to absolute path
+    local allowed_dir
+    allowed_dir=$(cd "$base_dir" 2>/dev/null && pwd) || {
+        xml_json_err "INVALID_BASE_DIR" \
+            "Base directory does not exist" \
+            "dir=$base_dir"
+        return 1
+    }
+
+    # First check: reject paths with traversal sequences
+    if [[ "$include_path" =~ \.\.[\/] ]] || [[ "$include_path" =~ [\/]\.\. ]]; then
+        xml_json_err "PATH_TRAVERSAL_DETECTED" \
+            "Path contains traversal sequences" \
+            "path=$include_path"
+        return 1
+    fi
+
+    # Build resolved path
+    local resolved_path
+    if [[ "$include_path" == /* ]]; then
+        # Absolute path - must start with allowed_dir
+        if [[ ! "$include_path" =~ ^"$allowed_dir" ]]; then
+            xml_json_err "PATH_TRAVERSAL_BLOCKED" \
+                "Absolute path outside allowed directory" \
+                "path=$include_path, allowed=$allowed_dir"
+            return 1
+        fi
+        resolved_path="$include_path"
+    else
+        # Relative path - resolve against base_dir
+        resolved_path="$allowed_dir/$include_path"
+    fi
+
+    # Normalize path (remove . and .. components manually for portability)
+    local normalized_path="$resolved_path"
+
+    # Verify final path is within allowed directory
+    if [[ ! "$normalized_path" =~ ^"$allowed_dir" ]]; then
+        xml_json_err "PATH_TRAVERSAL_BLOCKED" \
+            "Resolved path outside allowed directory" \
+            "path=$normalized_path, allowed=$allowed_dir"
+        return 1
+    fi
+
+    echo "$normalized_path"
+}
+
+# ============================================================================
+# XInclude Composition Functions
+# ============================================================================
+
+# xml-compose: Resolve XInclude directives in worker priming documents
+# Usage: xml-compose <input_xml> [output_xml]
+# Returns: {"ok":true,"result":{"composed":true,"output":"...","sources_resolved":N}}
+xml-compose() {
+    local input_xml="${1:-}"
+    local output_xml="${2:-}"
+
+    [[ -z "$input_xml" ]] && { xml_json_err "Missing input XML file argument"; return 1; }
+    [[ -f "$input_xml" ]] || { xml_json_err "Input XML file not found: $input_xml"; return 1; }
+
+    # Check well-formedness first
+    local well_formed_result
+    well_formed_result=$(xml-well-formed "$input_xml" 2>/dev/null)
+    if ! echo "$well_formed_result" | jq -e '.result.well_formed' >/dev/null 2>&1; then
+        xml_json_err "Input XML is not well-formed"
+        return 1
+    fi
+
+    # Use xmllint for XInclude processing if available (with XXE protection)
+    if [[ "$XMLLINT_AVAILABLE" == "true" ]]; then
+        local composed
+        composed=$(xmllint --nonet --noent --xinclude --format "$input_xml" 2>/dev/null) || {
+            xml_json_err "XInclude composition failed - check that included files exist"
+            return 1
+        }
+
+        # Determine output destination
+        if [[ -n "$output_xml" ]]; then
+            echo "$composed" > "$output_xml"
+            local escaped_output
+            escaped_output=$(echo "$output_xml" | jq -Rs '.[:-1]')
+            xml_json_ok "{\"composed\":true,\"output\":$escaped_output,\"sources_resolved\":\"auto\"}"
+        else
+            # Output to stdout wrapped in JSON
+            local escaped_composed
+            escaped_composed=$(echo "$composed" | jq -Rs '.')
+            xml_json_ok "{\"composed\":true,\"xml\":$escaped_composed,\"sources_resolved\":\"auto\"}"
+        fi
+        return 0
+    fi
+
+    # No xmllint available - error explicitly (manual fallback removed for security)
+    xml_json_err "XMLLINT_REQUIRED" \
+        "xmllint is required for secure XInclude processing" \
+        "install_hint='brew install libxml2'"
+    return 1
+}
+
+# xml-list-includes: List all XInclude references in a document
+# Usage: xml-list-includes <xml_file>
+# Returns: {"ok":true,"result":{"includes":[{"href":"...","parse":"..."},...]}}
+xml-list-includes() {
+    local xml_file="${1:-}"
+
+    [[ -z "$xml_file" ]] && { xml_json_err "Missing XML file argument"; return 1; }
+    [[ -f "$xml_file" ]] || { xml_json_err "XML file not found: $xml_file"; return 1; }
+
+    local includes_json="[]"
+
+    if [[ "$XMLSTARLET_AVAILABLE" == "true" ]]; then
+        # Use xmlstarlet for proper namespace-aware extraction
+        includes_json=$(xmlstarlet sel -N xi="http://www.w3.org/2001/XInclude" \
+            -t -m "//xi:include" \
+            -o '{"href":"' -v "@href" -o '","parse":"' -v "@parse" -o '","xpointer":"' -v "@xpointer" -o '"}' \
+            -n "$xml_file" 2>/dev/null | jq -s '.' || echo "[]")
+    elif [[ "$XMLLINT_AVAILABLE" == "true" ]]; then
+        # Fallback: grep for xi:include (less reliable but portable)
+        local base_dir
+        base_dir=$(dirname "$xml_file")
+
+        includes_json=$(grep -oE 'xi:include[^>]*href="[^"]+"' "$xml_file" 2>/dev/null | while read -r match; do
+            local href
+            href=$(echo "$match" | grep -oE 'href="[^"]+"' | cut -d'"' -f2)
+            local parse
+            parse=$(echo "$match" | grep -oE 'parse="[^"]+"' | cut -d'"' -f2 || echo "xml")
+            echo "{\"href\":\"$href\",\"parse\":\"$parse\",\"resolved\":\"$base_dir/$href\"}"
+        done | jq -s '.' || echo "[]")
+    else
+        xml_json_err "No XML tool available. Install xmlstarlet or libxml2."
+        return 1
+    fi
+
+    local count
+    count=$(echo "$includes_json" | jq 'length')
+    xml_json_ok "{\"includes\":$includes_json,\"count\":$count}"
+}
+
+# xml-compose-worker-priming: Specialized composition for worker priming documents
+# Usage: xml-compose-worker-priming <priming_xml> [output_xml]
+# Returns: {"ok":true,"result":{"composed":true,"worker_id":"...","caste":"...","sources":{...}}}
+xml-compose-worker-priming() {
+    local priming_xml="${1:-}"
+    local output_xml="${2:-}"
+
+    [[ -z "$priming_xml" ]] && { xml_json_err "Missing priming XML file argument"; return 1; }
+    [[ -f "$priming_xml" ]] || { xml_json_err "Priming XML file not found: $priming_xml"; return 1; }
+
+    # Validate against schema if available
+    local schema_file=".aether/schemas/worker-priming.xsd"
+    if [[ -f "$schema_file" ]] && [[ "$XMLLINT_AVAILABLE" == "true" ]]; then
+        local validation
+        validation=$(xml-validate "$priming_xml" "$schema_file" 2>/dev/null)
+        if ! echo "$validation" | jq -e '.result.valid' >/dev/null 2>&1; then
+            xml_json_err "Worker priming XML failed schema validation"
+            return 1
+        fi
+    fi
+
+    # Extract worker identity before composition
+    local worker_id worker_caste
+    if [[ "$XMLSTARLET_AVAILABLE" == "true" ]]; then
+        worker_id=$(xmlstarlet sel -t -v "//*[local-name()='worker-identity']/@id" "$priming_xml" 2>/dev/null || echo "unknown")
+        worker_caste=$(xmlstarlet sel -t -v "//*[local-name()='worker-identity']/*[local-name()='caste']" "$priming_xml" 2>/dev/null || echo "unknown")
+    else
+        # Fallback: sed extraction (portable, no grep -P)
+        worker_id=$(sed -n 's/.*worker-identity[^>]*id="\([^"]*\)".*/\1/p' "$priming_xml" | head -1 || echo "unknown")
+        worker_caste=$(sed -n 's/.*<caste>\([^<]*\)<\/caste>.*/\1/p' "$priming_xml" | head -1 || echo "unknown")
+    fi
+
+    # Compose the document
+    local compose_result
+    if [[ -n "$output_xml" ]]; then
+        compose_result=$(xml-compose "$priming_xml" "$output_xml" 2>/dev/null)
+    else
+        compose_result=$(xml-compose "$priming_xml" 2>/dev/null)
+    fi
+
+    if ! echo "$compose_result" | jq -e '.ok' >/dev/null 2>&1; then
+        xml_json_err "Composition failed: $(echo "$compose_result" | jq -r '.error // "unknown"')"
+        return 1
+    fi
+
+    # Count sources from different sections
+    local queen_wisdom_count active_trails_count stack_profiles_count
+    if [[ "$XMLSTARLET_AVAILABLE" == "true" ]] && [[ -n "$output_xml" ]]; then
+        queen_wisdom_count=$(xmlstarlet sel -t -v "count(//*[local-name()='queen-wisdom']/*[local-name()='wisdom-source'])" "$output_xml" 2>/dev/null || echo "0")
+        active_trails_count=$(xmlstarlet sel -t -v "count(//*[local-name()='active-trails']/*[local-name()='trail-source'])" "$output_xml" 2>/dev/null || echo "0")
+        stack_profiles_count=$(xmlstarlet sel -t -v "count(//*[local-name()='stack-profiles']/*[local-name()='profile-source'])" "$output_xml" 2>/dev/null || echo "0")
+    else
+        queen_wisdom_count="unknown"
+        active_trails_count="unknown"
+        stack_profiles_count="unknown"
+    fi
+
+    # Build result
+    local result_json
+    result_json=$(jq -n \
+        --arg worker_id "$worker_id" \
+        --arg caste "$worker_caste" \
+        --arg queen_wisdom "$queen_wisdom_count" \
+        --arg active_trails "$active_trails_count" \
+        --arg stack_profiles "$stack_profiles_count" \
+        '{
+            composed: true,
+            worker_id: $worker_id,
+            caste: $caste,
+            sources: {
+                queen_wisdom: $queen_wisdom,
+                active_trails: $active_trails,
+                stack_profiles: $stack_profiles
+            }
+        }')
+
+    xml_json_ok "$result_json"
+}
+
+# Export functions
+export -f xml-compose xml-list-includes xml-compose-worker-priming
+export -f xml-validate-include-path

package/.aether/utils/xml-convert.sh

@@ -0,0 +1,273 @@
+#!/bin/bash
+# XML Conversion Utilities
+# Bidirectional JSON/XML conversion and document merging
+#
+# Usage: source .aether/utils/xml-convert.sh
+#        xml-to-json <xml_file> [--pretty]
+#        json-to-xml <json_file> [root_element]
+#        xml-merge <output_file> <main_xml_file>
+#        xml-convert-detect-format <file>
+
+set -euo pipefail
+
+# Source xml-core.sh for JSON helpers and tool detection
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/xml-core.sh"
+
+# Additional tool detection for conversion
+XML2JSON_AVAILABLE=false
+if command -v xml2json >/dev/null 2>&1; then
+    XML2JSON_AVAILABLE=true
+fi
+
+# ============================================================================
+# Format Detection
+# ============================================================================
+
+# xml-convert-detect-format: Detect if file is XML or JSON
+# Usage: xml-convert-detect-format <file>
+# Returns: {"ok":true,"result":{"format":"xml|json|unknown","confidence":"high|medium|low"}}
+xml-convert-detect-format() {
+    local file="${1:-}"
+
+    [[ -z "$file" ]] && { xml_json_err "MISSING_ARG" "Missing file argument"; return 1; }
+    [[ -f "$file" ]] || { xml_json_err "FILE_NOT_FOUND" "File not found: $file"; return 1; }
+
+    # Read first 1KB for analysis
+    local header
+    header=$(head -c 1024 "$file" 2>/dev/null || head -c 1024 < "$file")
+
+    # Check for XML signatures
+    if echo "$header" | grep -qE '^\s*<\?xml\s+version'; then
+        xml_json_ok '{"format":"xml","confidence":"high","signature":"xml_declaration"}'
+        return 0
+    fi
+
+    if echo "$header" | grep -qE '^\s*<[a-zA-Z_][a-zA-Z0-9_]*[\s>]'; then
+        xml_json_ok '{"format":"xml","confidence":"medium","signature":"root_element"}'
+        return 0
+    fi
+
+    # Check for JSON signatures
+    if echo "$header" | grep -qE '^\s*(\{|\[)'; then
+        # Verify it's valid JSON
+        if jq empty "$file" 2>/dev/null; then
+            xml_json_ok '{"format":"json","confidence":"high","signature":"valid_json"}'
+            return 0
+        else
+            xml_json_ok '{"format":"json","confidence":"low","signature":"json_like","note":"Invalid JSON syntax"}'
+            return 0
+        fi
+    fi
+
+    xml_json_ok '{"format":"unknown","confidence":"low","signature":"none"}'
+}
+
+# ============================================================================
+# XML to JSON Conversion
+# ============================================================================
+
+# xml-to-json: Convert XML to JSON format
+# Usage: xml-to-json <xml_file> [--pretty]
+# Returns: {"ok":true,"result":{"json":"...","format":"object"}}
+xml-to-json() {
+    local xml_file="${1:-}"
+    local pretty=false
+
+    # Parse optional arguments
+    shift || true
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --pretty) pretty=true; shift ;;
+            *) shift ;;
+        esac
+    done
+
+    [[ -z "$xml_file" ]] && { xml_json_err "MISSING_ARG" "Missing XML file argument"; return 1; }
+    [[ -f "$xml_file" ]] || { xml_json_err "FILE_NOT_FOUND" "XML file not found: $xml_file"; return 1; }
+
+    # Check well-formedness first
+    local well_formed_result
+    well_formed_result=$(xml-well-formed "$xml_file" 2>/dev/null)
+    if ! echo "$well_formed_result" | jq -e '.result.well_formed' >/dev/null 2>&1; then
+        xml_json_err "PARSE_ERROR" "XML is not well-formed"
+        return 1
+    fi
+
+    # Try xml2json if available (npm package)
+    if [[ "$XML2JSON_AVAILABLE" == "true" ]]; then
+        local json_output
+        if json_output=$(xml2json "$xml_file" 2>/dev/null); then
+            if [[ "$pretty" == "true" ]]; then
+                json_output=$(echo "$json_output" | jq '.')
+            fi
+            local escaped_json
+            escaped_json=$(echo "$json_output" | jq -Rs '.')
+            xml_json_ok "{\"format\":\"object\",\"json\":$escaped_json}"
+            return 0
+        fi
+    fi
+
+    # Fallback: Use xsltproc with built-in XSLT
+    if [[ "$XSLTPROC_AVAILABLE" == "true" ]]; then
+        local xslt_script
+        xslt_script=$(cat << 'XSLT'
+<?xml version="1.0"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+<xsl:output method="text"/>
+<xsl:template match="/">
+<xsl:text>{"root":</xsl:text>
+<xsl:apply-templates select="*"/>
+<xsl:text>}</xsl:text>
+</xsl:template>
+<xsl:template match="*">
+<xsl:text>{"</xsl:text>
+<xsl:value-of select="name()"/>
+<xsl:text>":</xsl:text>
+<xsl:choose>
+<xsl:when test="count(*) > 0">
+<xsl:text>[</xsl:text>
+<xsl:apply-templates select="*"/>
+<xsl:text>]</xsl:text>
+</xsl:when>
+<xsl:otherwise>
+<xsl:text>"</xsl:text>
+<xsl:value-of select="."/>
+<xsl:text>"</xsl:text>
+</xsl:otherwise>
+</xsl:choose>
+<xsl:text>}</xsl:text>
+<xsl:if test="position() != last()">,</xsl:if>
+</xsl:template>
+</xsl:stylesheet>
+XSLT
+)
+        local json_result
+        json_result=$(echo "$xslt_script" | xsltproc - "$xml_file" 2>/dev/null) || {
+            xml_json_err "CONVERSION_ERROR" "XSLT conversion failed"
+            return 1
+        }
+        xml_json_ok "{\"format\":\"object\",\"json\":$(echo "$json_result" | jq -Rs '.')}"
+        return 0
+    fi
+
+    # Last resort: Use xmlstarlet if available
+    if [[ "$XMLSTARLET_AVAILABLE" == "true" ]]; then
+        local json_result
+        json_result=$(xmlstarlet sel -t -m "/" -o '{"root":{' -m "*" -v "name()" -o ':"' -v "." -o '"' -b -o '}}' "$xml_file" 2>/dev/null) || {
+            xml_json_err "CONVERSION_ERROR" "xmlstarlet conversion failed"
+            return 1
+        }
+        xml_json_ok "{\"format\":\"object\",\"json\":$(echo "$json_result" | jq -Rs '.')}"
+        return 0
+    fi
+
+    xml_json_err "TOOL_NOT_AVAILABLE" "No XML to JSON conversion tool available. Install xml2json, xsltproc, or xmlstarlet."
+    return 1
+}
+
+# ============================================================================
+# JSON to XML Conversion
+# ============================================================================
+
+# json-to-xml: Convert JSON to XML
+# Usage: json-to-xml <json_file> [root_element]
+# Returns: {"ok":true,"result":{"xml":"<root>...</root>"}}
+json-to-xml() {
+    local json_file="${1:-}"
+    local root_element="${2:-root}"
+
+    [[ -z "$json_file" ]] && { xml_json_err "MISSING_ARG" "Missing JSON file argument"; return 1; }
+    [[ -f "$json_file" ]] || { xml_json_err "FILE_NOT_FOUND" "JSON file not found: $json_file"; return 1; }
+
+    # Validate JSON first
+    if ! jq empty "$json_file" 2>/dev/null; then
+        xml_json_err "PARSE_ERROR" "Invalid JSON file: $json_file"
+        return 1
+    fi
+
+    # Build XML using jq to generate structure
+    local xml_output
+    xml_output=$(jq -r --arg root "$root_element" '
+        def to_xml:
+            if type == "object" then
+                to_entries | map(
+                    "<\(.key)>\(.value | to_xml)</\(.key)>"
+                ) | join("")
+            elif type == "array" then
+                map("<item>\(. | to_xml)</item>") | join("")
+            elif type == "string" then
+                .
+            elif type == "number" then
+                tostring
+            elif type == "boolean" then
+                tostring
+            elif type == "null" then
+                ""
+            else
+                tostring
+            end;
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<\($root)>\n" + (to_xml) + "\n</\($root)>"
+    ' "$json_file" 2>/dev/null) || {
+        xml_json_err "CONVERSION_ERROR" "JSON to XML conversion failed"
+        return 1
+    }
+
+    # Escape the XML for JSON output
+    local escaped_xml
+    escaped_xml=$(echo "$xml_output" | jq -Rs '.')
+    xml_json_ok "{\"xml\":$escaped_xml}"
+}
+
+# ============================================================================
+# XML Document Merging
+# ============================================================================
+
+# xml-merge: XInclude document merging
+# Usage: xml-merge <output_file> <main_xml_file>
+# Returns: {"ok":true,"result":{"merged":true,"output":"<path>","sources_resolved":N}}
+xml-merge() {
+    local output_file="${1:-}"
+    local main_xml="${2:-}"
+
+    [[ -z "$output_file" ]] && { xml_json_err "MISSING_ARG" "Missing output file argument"; return 1; }
+    [[ -z "$main_xml" ]] && { xml_json_err "MISSING_ARG" "Missing main XML file argument"; return 1; }
+    [[ -f "$main_xml" ]] || { xml_json_err "FILE_NOT_FOUND" "Main XML file not found: $main_xml"; return 1; }
+
+    # Check well-formedness of main file
+    local well_formed_result
+    well_formed_result=$(xml-well-formed "$main_xml" 2>/dev/null)
+    if ! echo "$well_formed_result" | jq -e '.result.well_formed' >/dev/null 2>&1; then
+        xml_json_err "PARSE_ERROR" "Main XML file is not well-formed"
+        return 1
+    fi
+
+    # Use xmllint for XInclude processing with security flags
+    if [[ "$XMLLINT_AVAILABLE" == "true" ]]; then
+        local merged
+        merged=$(xmllint --nonet --noent --xinclude "$main_xml" 2>/dev/null) || {
+            xml_json_err "MERGE_ERROR" "XInclude merge failed"
+            return 1
+        }
+
+        # Write output
+        echo "$merged" > "$output_file"
+
+        # Count resolved includes (approximate)
+        local resolved_count
+        resolved_count=$(echo "$merged" | grep -c '<xi:include' 2>/dev/null || echo "0")
+
+        local escaped_output
+        escaped_output=$(echo "$output_file" | jq -Rs '.[:-1]')
+        xml_json_ok "{\"merged\":true,\"output\":$escaped_output,\"sources_resolved\":$resolved_count}"
+        return 0
+    fi
+
+    # Without xmllint, we cannot safely process XInclude
+    xml_json_err "TOOL_NOT_AVAILABLE" "xmllint required for XInclude merging. Install libxml2 utilities."
+    return 1
+}
+
+# Export functions
+export -f xml-convert-detect-format xml-to-json json-to-xml xml-merge
+export XML2JSON_AVAILABLE