aether-colony 5.0.0 → 5.2.1

package/.aether/templates/learning-observations.template.json

@@ -0,0 +1,6 @@
+{
+  "_template": "learning-observations",
+  "_version": "1.0",
+  "_instructions": "Write to .aether/data/learning-observations.json. No substitution needed. Remove underscore-prefixed keys.",
+  "observations": []
+}

package/.aether/templates/midden.template.json

@@ -0,0 +1,13 @@
+{
+  "_template": "midden",
+  "_version": "1.0",
+  "_instructions": "Write to .aether/data/midden/midden.json. No substitution needed. Remove underscore-prefixed keys.",
+  "version": "1.0.0",
+  "signals": [],
+  "spawn_metrics": {
+    "total_spawned": 0,
+    "completed": 0,
+    "failed": 0,
+    "efficiency_pct": 0
+  }
+}

package/.aether/templates/pheromones.template.json

@@ -0,0 +1,6 @@
+{
+  "_template": "pheromones",
+  "_version": "1.0",
+  "_instructions": "Write to .aether/data/pheromones.json. No substitution needed. Remove underscore-prefixed keys.",
+  "signals": []
+}

package/.aether/templates/session.template.json

@@ -0,0 +1,9 @@
+{
+  "_template": "session",
+  "_version": "1.0",
+  "_instructions": "Write to .aether/data/session.json. Replace __SESSION_ID__, __GOAL__, __TIMESTAMP__. Remove underscore-prefixed keys.",
+  "session_id": "__SESSION_ID__",
+  "colony_goal": "__GOAL__",
+  "started_at": "__TIMESTAMP__",
+  "last_activity": "__TIMESTAMP__"
+}

package/.aether/utils/atomic-write.sh

@@ -25,11 +25,13 @@ if [ ! -f "$_AETHER_UTILS_DIR/file-lock.sh" ]; then
 fi
 source "$_AETHER_UTILS_DIR/file-lock.sh"
 
-# Aether root detection - use git root if available, otherwise use current directory
-if git rev-parse --show-toplevel >/dev/null 2>&1; then
-    AETHER_ROOT="$(git rev-parse --show-toplevel)"
-else
-    AETHER_ROOT="$(pwd)"
+# Aether root detection - respect existing AETHER_ROOT, or use git root, or use current directory
+if [[ -z "${AETHER_ROOT:-}" ]]; then
+    if git rev-parse --show-toplevel >/dev/null 2>&1; then
+        AETHER_ROOT="$(git rev-parse --show-toplevel)"
+    else
+        AETHER_ROOT="$(pwd)"
+    fi
 fi
 
 TEMP_DIR="$AETHER_ROOT/.aether/temp"
@@ -41,9 +43,44 @@ mkdir -p "$TEMP_DIR" "$BACKUP_DIR"
 # Number of backups to keep
 MAX_BACKUPS=3
 
+# Safety stats file for tracking data safety events (best-effort, never fails operations)
+SAFETY_STATS_FILE="${AETHER_ROOT}/.aether/data/safety-stats.json"
+
+# Increment a safety stats counter (best-effort, never fails the calling operation)
+# Arguments: counter_name (e.g., "stale_locks_cleaned", "json_validation_rejects")
+_safety_stats_increment() {
+    local counter_name="$1"
+    local stats_file="$SAFETY_STATS_FILE"
+    local now_iso
+    now_iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "unknown")
+
+    # Ensure data directory exists
+    mkdir -p "$(dirname "$stats_file")" 2>/dev/null || return 0
+
+    # Initialize if missing
+    if [[ ! -f "$stats_file" ]]; then
+        printf '{"stale_locks_cleaned":0,"json_validation_rejects":0,"last_updated":"%s"}\n' "$now_iso" > "$stats_file" 2>/dev/null || return 0
+    fi
+
+    # Increment counter (best-effort, don't fail on jq errors)
+    local updated
+    updated=$(jq --arg key "$counter_name" --arg ts "$now_iso" '
+        .[$key] = ((.[$key] // 0) + 1) |
+        .last_updated = $ts
+    ' "$stats_file" 2>/dev/null) || return 0
+
+    if [[ -n "$updated" ]]; then
+        printf '%s\n' "$updated" > "$stats_file" 2>/dev/null || return 0
+    fi
+}
+
 # Atomic write: write content to file via temporary file
 # Arguments: target_file, content
 # Returns: 0 on success, 1 on failure
+# NOTE: atomic_write does NOT interact with file locks. Lock management
+# (acquire_lock/release_lock) is the CALLER's responsibility. If you need
+# exclusive access, acquire the lock before calling atomic_write, and release
+# it after (including on error paths).
 atomic_write() {
     local target_file="$1"
     local content="$2"
@@ -53,10 +90,10 @@ atomic_write() {
     mkdir -p "$target_dir"
 
     # Create unique temp file
-    local temp_file="${TEMP_DIR}/$(basename "$target_file").$$.$(date +%s%N).tmp"
+    local temp_file="${TEMP_DIR}/$(basename "$target_file").$$.$( date +%s )_${RANDOM}.tmp"
 
     # Write content to temp file
-    if ! echo "$content" > "$temp_file"; then
+    if ! printf '%s\n' "$content" > "$temp_file"; then
         echo "Failed to write to temp file: $temp_file"
         rm -f "$temp_file"
         return 1
@@ -67,11 +104,12 @@ atomic_write() {
         create_backup "$target_file"
     fi
 
-    # Validate JSON if it's a JSON file
+    # Validate JSON if it's a JSON file (lock management is caller's responsibility)
     if [[ "$target_file" == *.json ]]; then
-        if ! python3 -c "import json; json.load(open('$temp_file'))" 2>/dev/null; then
+        if ! jq empty "$temp_file" 2>/dev/null; then
             echo "Invalid JSON in temp file: $temp_file"
             rm -f "$temp_file"
+            _safety_stats_increment "json_validation_rejects" 2>/dev/null || true
             return 1
         fi
     fi
@@ -108,7 +146,7 @@ atomic_write_from_file() {
     mkdir -p "$target_dir"
 
     # Create unique temp file
-    local temp_file="${TEMP_DIR}/$(basename "$target_file").$$.$(date +%s%N).tmp"
+    local temp_file="${TEMP_DIR}/$(basename "$target_file").$$.$( date +%s )_${RANDOM}.tmp"
 
     # Copy source to temp
     if ! cp "$source_file" "$temp_file"; then
@@ -122,11 +160,12 @@ atomic_write_from_file() {
         create_backup "$target_file"
     fi
 
-    # Validate JSON if it's a JSON file
+    # Validate JSON if it's a JSON file (lock management is caller's responsibility)
     if [[ "$target_file" == *.json ]]; then
-        if ! python3 -c "import json; json.load(open('$temp_file'))" 2>/dev/null; then
+        if ! jq empty "$temp_file" 2>/dev/null; then
             echo "Invalid JSON in temp file: $temp_file"
             rm -f "$temp_file"
+            _safety_stats_increment "json_validation_rejects" 2>/dev/null || true
             return 1
         fi
     fi
@@ -166,10 +205,17 @@ create_backup() {
 # Arguments: base_name
 rotate_backups() {
     local base_name="$1"
-    local backups=$(ls -t "${BACKUP_DIR}/${base_name}".*.backup 2>/dev/null | wc -l)
 
-    if [ "$backups" -gt "$MAX_BACKUPS" ]; then
-        ls -t "${BACKUP_DIR}/${base_name}".*.backup | tail -n +$((MAX_BACKUPS + 1)) | xargs rm -f
+    # Use find with -print0 for safe handling of paths with spaces
+    local backup_count
+    backup_count=$(find "$BACKUP_DIR" -maxdepth 1 -name "${base_name}.*.backup" -type f 2>/dev/null | wc -l | tr -d ' ')
+
+    if [ "$backup_count" -gt "$MAX_BACKUPS" ]; then
+        # Delete oldest backups beyond MAX_BACKUPS using find for space-safe handling
+        find "$BACKUP_DIR" -maxdepth 1 -name "${base_name}.*.backup" -type f -print0 2>/dev/null \
+            | xargs -0 ls -t 2>/dev/null \
+            | tail -n +$((MAX_BACKUPS + 1)) \
+            | while IFS= read -r file; do rm -f "$file" 2>/dev/null || true; done
     fi
 }
 
@@ -209,9 +255,9 @@ list_backups() {
 
 # Cleanup temp files older than 1 hour
 cleanup_temp_files() {
-    find "$TEMP_DIR" -name "*.tmp" -mtime +1/24 -delete 2>/dev/null || true
+    find "$TEMP_DIR" -name "*.tmp" -mmin +60 -delete 2>/dev/null || true
 }
 
 # Export functions
 export -f atomic_write atomic_write_from_file create_backup rotate_backups
-export -f restore_backup list_backups cleanup_temp_files
+export -f restore_backup list_backups cleanup_temp_files _safety_stats_increment

package/.aether/utils/chamber-utils.sh

@@ -17,7 +17,10 @@ CURRENT_LOCK=${CURRENT_LOCK:-""}
 
 # Get script directory for sourcing (preserve parent SCRIPT_DIR if set)
 __chamber_utils_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-AETHER_ROOT="$(cd "$__chamber_utils_dir/../.." && pwd 2>/dev/null || echo "$__chamber_utils_dir/../..")"
+# Respect existing AETHER_ROOT if already set
+if [[ -z "${AETHER_ROOT:-}" ]]; then
+    AETHER_ROOT="$(cd "$__chamber_utils_dir/../.." && pwd 2>/dev/null || echo "$__chamber_utils_dir/../..")"
+fi
 
 # Use parent SCRIPT_DIR if available, otherwise use local
 SCRIPT_DIR="${SCRIPT_DIR:-$__chamber_utils_dir}"
@@ -44,6 +47,7 @@ fi
 : "${E_FILE_NOT_FOUND:=E_FILE_NOT_FOUND}"
 : "${E_BASH_ERROR:=E_BASH_ERROR}"
 : "${E_JSON_INVALID:=E_JSON_INVALID}"
+: "${E_FEATURE_UNAVAILABLE:=E_FEATURE_UNAVAILABLE}"
 
 # --- Chamber Functions ---
 
@@ -293,5 +297,144 @@ EOF
   json_ok "$sorted"
 }
 
+# --- Colony Archive XML ---
+
+# Export combined colony archive XML containing pheromones, wisdom, and registry
+# Usage: _colony_archive_xml [output_file]
+# Default output: .aether/exchange/colony-archive.xml
+# Always filters to active-only pheromone signals
+_colony_archive_xml() {
+    # Graceful degradation: check for xmllint
+    if ! command -v xmllint >/dev/null 2>&1; then
+      json_err "$E_FEATURE_UNAVAILABLE" "xmllint is not installed. Try: xcode-select --install on macOS."
+    fi
+
+    cax_output="${1:-$SCRIPT_DIR/exchange/colony-archive.xml}"
+    mkdir -p "$(dirname "$cax_output")"
+
+    # Step 1: Filter active-only pheromone signals to a temp file
+    cax_tmp_pheromones=$(mktemp)
+    if [[ -f "$COLONY_DATA_DIR/pheromones.json" ]]; then
+      jq '{
+        version: .version,
+        colony_id: .colony_id,
+        generated_at: .generated_at,
+        signals: [.signals[] | select(.active == true)]
+      }' "$COLONY_DATA_DIR/pheromones.json" > "$cax_tmp_pheromones" 2>/dev/null  # SUPPRESS:OK -- read-default: file may not exist yet
+    else
+      printf '%s\n' '{"version":"1.0","colony_id":"unknown","generated_at":"","signals":[]}' > "$cax_tmp_pheromones"
+    fi
+
+    # Step 2: Export each section to temp XML files
+    cax_tmp_dir=$(mktemp -d)
+
+    # Pheromone section (using filtered active-only)
+    source "$SCRIPT_DIR/exchange/pheromone-xml.sh"
+    xml-pheromone-export "$cax_tmp_pheromones" "$cax_tmp_dir/pheromones.xml" 2>/dev/null || _aether_log_error "Could not export pheromones to XML"
+
+    # Wisdom section — reuse wisdom-export-xml fallback logic
+    source "$SCRIPT_DIR/exchange/wisdom-xml.sh"
+    cax_wisdom_input="$DATA_DIR/queen-wisdom.json"
+    if [[ ! -f "$cax_wisdom_input" ]]; then
+      # MIGRATE: direct COLONY_STATE.json access -- use _state_read_field instead
+      # Try extracting from COLONY_STATE.json memory field
+      if [[ -f "$DATA_DIR/COLONY_STATE.json" ]]; then
+        cax_wex_memory=$(jq '.memory // {}' "$DATA_DIR/COLONY_STATE.json" 2>/dev/null || echo '{}')  # SUPPRESS:OK -- read-default: returns fallback if missing
+        if [[ "$cax_wex_memory" != "{}" && "$cax_wex_memory" != "null" ]]; then
+          cax_wex_created_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          cax_wisdom_input="$cax_tmp_dir/wisdom-input.json"
+          printf '%s\n' "{
+  \"version\": \"1.0.0\",
+  # SUPPRESS:OK -- read-default: query may return empty
+  \"metadata\": {\"created\": \"$cax_wex_created_at\", \"colony_id\": \"$(jq -r '.goal // \"unknown\"' "$DATA_DIR/COLONY_STATE.json" 2>/dev/null)\"},
+  \"philosophies\": [],
+  # SUPPRESS:OK -- read-default: query may return empty
+  \"patterns\": $(echo "$cax_wex_memory" | jq '[.instincts // [] | .[] | {"id": (. | @base64), "content": ., "confidence": 0.7, "domain": "general", "source": "colony_memory"}]' 2>/dev/null || echo '[]')
+}" > "$cax_wisdom_input"
+        fi
+      fi
+    fi
+    if [[ -f "$cax_wisdom_input" ]]; then
+      xml-wisdom-export "$cax_wisdom_input" "$cax_tmp_dir/wisdom.xml" 2>/dev/null || _aether_log_error "Could not export wisdom to XML"
+    fi
+
+    # Registry section — reuse registry-export-xml on-demand generation logic
+    source "$SCRIPT_DIR/exchange/registry-xml.sh"
+    cax_registry_input="$DATA_DIR/colony-registry.json"
+    if [[ ! -f "$cax_registry_input" ]]; then
+      cax_rex_chambers_dir="$AETHER_ROOT/.aether/chambers"
+      cax_rex_generated_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+      cax_rex_colonies="[]"
+      if [[ -d "$cax_rex_chambers_dir" ]]; then
+        cax_rex_colonies=$(
+          for manifest in "$cax_rex_chambers_dir"/*/manifest.json; do
+            [[ -f "$manifest" ]] || continue
+            jq -c '{
+              id: (.colony_id // .goal // "unknown"),
+              name: (.goal // "Unnamed Colony"),
+              created_at: (.created_at // "unknown"),
+              sealed_at: (.sealed_at // null),
+              status: (if .sealed_at then "sealed" else "active" end),
+              chamber: input_filename
+            }' "$manifest" 2>/dev/null || true  # SUPPRESS:OK -- cleanup: operation is best-effort
+          done | jq -s '.' 2>/dev/null || echo '[]'  # SUPPRESS:OK -- read-default: returns fallback if missing
+        )
+      fi
+      cax_registry_input="$cax_tmp_dir/registry-input.json"
+      printf '%s\n' "{
+  \"version\": \"1.0.0\",
+  \"generated_at\": \"$cax_rex_generated_at\",
+  \"colonies\": $cax_rex_colonies
+}" > "$cax_registry_input"
+    fi
+    xml-registry-export "$cax_registry_input" "$cax_tmp_dir/registry.xml" 2>/dev/null || _aether_log_error "Could not export registry to XML"
+
+    # Step 3: Build combined XML
+    # SUPPRESS:OK -- read-default: query may return empty
+    cax_colony_id=$(jq -r '.goal // "unknown"' "$DATA_DIR/COLONY_STATE.json" 2>/dev/null | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//;s/-$//')
+    [[ -z "$cax_colony_id" || "$cax_colony_id" == "unknown" ]] && cax_colony_id="unknown"
+    cax_sealed_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    cax_pheromone_count=$(jq '.signals | length' "$cax_tmp_pheromones" 2>/dev/null || echo 0)  # SUPPRESS:OK -- read-default: file may not exist yet
+
+    {
+      printf '<?xml version="1.0" encoding="UTF-8"?>\n'
+      printf '<colony-archive\n'
+      printf '    xmlns="http://aether.colony/schemas/archive/1.0"\n'
+      printf '    colony_id="%s"\n' "$cax_colony_id"
+      printf '    sealed_at="%s"\n' "$cax_sealed_at"
+      printf '    version="1.0.0"\n'
+      printf '    pheromone_count="%s">\n' "$cax_pheromone_count"
+
+      # Append pheromone section (strip XML declaration)
+      if [[ -f "$cax_tmp_dir/pheromones.xml" ]]; then
+        sed '1{/^<?xml/d;}' "$cax_tmp_dir/pheromones.xml"
+      fi
+
+      # Append wisdom section (strip XML declaration)
+      if [[ -f "$cax_tmp_dir/wisdom.xml" ]]; then
+        sed '1{/^<?xml/d;}' "$cax_tmp_dir/wisdom.xml"
+      fi
+
+      # Append registry section (strip XML declaration)
+      if [[ -f "$cax_tmp_dir/registry.xml" ]]; then
+        sed '1{/^<?xml/d;}' "$cax_tmp_dir/registry.xml"
+      fi
+
+      printf '</colony-archive>\n'
+    } > "$cax_output"
+
+    # Step 4: Validate well-formedness
+    if xmllint --noout "$cax_output" 2>/dev/null; then  # SUPPRESS:OK -- validation: testing XML validity
+      cax_valid=true
+    else
+      cax_valid=false
+    fi
+
+    # Step 5: Cleanup temp files
+    rm -rf "$cax_tmp_dir" "$cax_tmp_pheromones"
+
+    json_ok "$(jq -n --arg path "$cax_output" --argjson valid "$cax_valid" --arg colony_id "$cax_colony_id" --argjson pheromone_count "$cax_pheromone_count" '{path: $path, valid: $valid, colony_id: $colony_id, pheromone_count: $pheromone_count}')"
+}
+
 # Export functions for use in other scripts
-export -f chamber_sanitize_goal chamber_compute_hash chamber_create chamber_verify chamber_list
+export -f chamber_sanitize_goal chamber_compute_hash chamber_create chamber_verify chamber_list _colony_archive_xml