@jaguilar87/gaia 5.0.8 → 5.0.9

package/tools/context/deep_merge.py

@@ -1,159 +0,0 @@
-"""
-Deep merge utility for project-context.json updates.
-
-Merges two dicts recursively following the gaia-ops merge decision tree:
-  1. Key missing in current  -> ADD
-  2. Both values are dicts   -> RECURSE (deep merge)
-  3. Both values are lists   -> UNION (primitives: sorted set union;
-                                       dicts with "name": merge by name;
-                                       other dicts: concatenate + deduplicate)
-  4. Both values are scalars -> OVERWRITE (new replaces old)
-  5. Type mismatch           -> OVERWRITE with warning
-
-No-Delete Policy: keys in current but NOT in update are always preserved.
-"""
-
-import copy
-import json
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def deep_merge(current: dict, update: dict) -> tuple[dict, dict]:
-    """Merge *update* into *current* returning ``(merged, diff)``.
-
-    Parameters
-    ----------
-    current:
-        The existing data (will NOT be mutated).
-    update:
-        New data to merge on top of *current*.
-
-    Returns
-    -------
-    tuple[dict, dict]
-        ``merged`` – the result of the merge.
-        ``diff``   – audit trail recording changes (``{key: {old, new}}``).
-    """
-    merged = copy.deepcopy(current)
-    diff: dict = {}
-
-    for key, new_value in update.items():
-        if key not in merged:
-            # Rule 1: ADD missing key
-            merged[key] = copy.deepcopy(new_value)
-            continue
-
-        old_value = merged[key]
-
-        # Rule 2: Both dicts -> recurse
-        if isinstance(old_value, dict) and isinstance(new_value, dict):
-            sub_merged, sub_diff = deep_merge(old_value, new_value)
-            merged[key] = sub_merged
-            if sub_diff:
-                diff[key] = sub_diff
-            continue
-
-        # Rule 3: Both lists -> union strategy
-        if isinstance(old_value, list) and isinstance(new_value, list):
-            merged_list = _merge_lists(old_value, new_value)
-            if merged_list != old_value:
-                diff[key] = {"old": old_value, "new": merged_list}
-            merged[key] = merged_list
-            continue
-
-        # Rule 5: Type mismatch -> overwrite with warning
-        if type(old_value) is not type(new_value):
-            logger.warning(
-                "Type mismatch for key '%s': %s -> %s. New value wins.",
-                key,
-                type(old_value).__name__,
-                type(new_value).__name__,
-            )
-            diff[key] = {"old": old_value, "new": new_value}
-            merged[key] = copy.deepcopy(new_value)
-            continue
-
-        # Rule 4: Both scalars -> overwrite
-        if old_value != new_value:
-            diff[key] = {"old": old_value, "new": new_value}
-        merged[key] = copy.deepcopy(new_value)
-
-    return merged, diff
-
-
-# ---------------------------------------------------------------------------
-# List merge helpers
-# ---------------------------------------------------------------------------
-
-def _merge_lists(current: list, update: list) -> list:
-    """Merge two lists following the union strategy.
-
-    a) All items are primitives (str, int, float, bool) -> sorted set union.
-    b) Items are dicts with a ``"name"`` key -> merge by name, preserve missing.
-    c) Otherwise -> concatenate, deduplicate by JSON equality.
-    """
-    if _all_primitives(current) and _all_primitives(update):
-        return sorted(set(current) | set(update))
-
-    if _all_dicts_with_name(current) and _all_dicts_with_name(update):
-        return _merge_named_dicts(current, update)
-
-    # Fallback: concatenate + deduplicate by JSON equality
-    return _concat_deduplicate(current, update)
-
-
-def _all_primitives(items: list) -> bool:
-    """Return True if every item is a primitive (str, int, float, bool)."""
-    return all(isinstance(i, (str, int, float, bool)) for i in items)
-
-
-def _all_dicts_with_name(items: list) -> bool:
-    """Return True if every item is a dict containing a ``"name"`` key."""
-    return bool(items) and all(
-        isinstance(i, dict) and "name" in i for i in items
-    )
-
-
-def _merge_named_dicts(current: list[dict], update: list[dict]) -> list[dict]:
-    """Merge lists of dicts by their ``"name"`` field.
-
-    - Matching names: deep-merge the dict fields.
-    - Names only in current: preserved (no-delete).
-    - Names only in update: appended.
-    """
-    result_by_name: dict[str, dict] = {}
-    order: list[str] = []
-
-    # Seed with current entries (preserves order + no-delete)
-    for item in current:
-        name = item["name"]
-        result_by_name[name] = copy.deepcopy(item)
-        order.append(name)
-
-    # Merge / add from update
-    for item in update:
-        name = item["name"]
-        if name in result_by_name:
-            merged_item, _ = deep_merge(result_by_name[name], item)
-            result_by_name[name] = merged_item
-        else:
-            result_by_name[name] = copy.deepcopy(item)
-            order.append(name)
-
-    return [result_by_name[n] for n in order]
-
-
-def _concat_deduplicate(current: list, update: list) -> list:
-    """Concatenate two lists, deduplicating by JSON equality."""
-    seen: list[str] = []
-    result: list = []
-
-    for item in current + update:
-        serialized = json.dumps(item, sort_keys=True)
-        if serialized not in seen:
-            seen.append(serialized)
-            result.append(copy.deepcopy(item))
-
-    return result

package/tools/migration/migrate_04_harness_events.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-"""
-migrate_04_harness_events.py
-
-Convierte events.jsonl -> archivo SQL con INSERT batched.
-
-Reglas:
-  - Solo I/O sobre filesystem.
-  - NO importa sqlite3.
-  - `id` es AUTOINCREMENT en la tabla; NO lo insertamos.
-  - `payload` = json.dumps(record) entero -- preserva todos los campos.
-  - Idempotencia: harness_events NO tiene UNIQUE constraint útil.
-    Aplicar una sola vez. Re-ejecuciones requieren DELETE WHERE project=...
-
-CLI args (parametrización cross-workspace):
-  --project   workspace name (default: 'me')
-  --src       path al events.jsonl (default: ws/me)
-  --out       path al SQL de salida (default: /tmp/migrate_04_harness_events.sql)
-  --fragment  emite solo INSERTs (sin BEGIN/COMMIT)
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from pathlib import Path
-
-DEFAULT_PROJECT = "me"
-DEFAULT_SRC = Path("/home/jorge/ws/me/.claude/events/events.jsonl")
-DEFAULT_OUT = Path("/tmp/migrate_04_harness_events.sql")
-BATCH_SIZE = 200
-
-COLUMNS = ["project", "ts", "type", "source", "agent", "result", "severity", "payload"]
-
-
-def sql_quote(value) -> str:
-    if value is None:
-        return "NULL"
-    if isinstance(value, bool):
-        return "1" if value else "0"
-    if isinstance(value, (int, float)):
-        if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))):
-            return "NULL"
-        return str(value)
-    s = str(value)
-    return "'" + s.replace("'", "''") + "'"
-
-
-def extract_row(record: dict, project: str) -> dict:
-    return {
-        "project": project,
-        "ts": record.get("ts"),
-        "type": record.get("type"),
-        "source": record.get("source"),
-        "agent": record.get("agent"),
-        "result": record.get("result"),
-        "severity": record.get("severity"),
-        "payload": json.dumps(record, ensure_ascii=False, separators=(",", ":")),
-    }
-
-
-def row_values_sql(row: dict) -> str:
-    return "(" + ",".join(sql_quote(row.get(col)) for col in COLUMNS) + ")"
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Generate INSERT SQL for harness_events table.")
-    parser.add_argument("--project", default=DEFAULT_PROJECT)
-    parser.add_argument("--src", default=str(DEFAULT_SRC), help="path to events.jsonl")
-    parser.add_argument("--out", default=str(DEFAULT_OUT))
-    parser.add_argument("--fragment", action="store_true")
-    args = parser.parse_args()
-
-    project = args.project
-    src = Path(args.src)
-    out = Path(args.out)
-    fragment = args.fragment
-
-    if not src.exists():
-        print(f"[migrate_04:{project}] ERROR: source not found: {src}", file=sys.stderr)
-        return 1
-
-    rows = []
-    skipped = 0
-    total_lines = 0
-
-    with src.open("r", encoding="utf-8") as f:
-        for line in f:
-            total_lines += 1
-            s = line.strip()
-            if not s:
-                continue
-            try:
-                rec = json.loads(s)
-            except json.JSONDecodeError:
-                skipped += 1
-                continue
-            if not rec.get("ts") or not rec.get("type"):
-                skipped += 1
-                continue
-            rows.append(extract_row(rec, project))
-
-    cols_csv = ",".join(COLUMNS)
-    insert_prefix = f"INSERT INTO harness_events ({cols_csv}) VALUES\n"
-
-    with out.open("w", encoding="utf-8") as fh:
-        fh.write(f"-- Generated by migrate_04_harness_events.py\n")
-        fh.write(f"-- Project: {project}\n")
-        fh.write(f"-- Source: {src}\n")
-        fh.write(f"-- Total source lines: {total_lines}\n")
-        fh.write(f"-- Records to insert:  {len(rows)}\n")
-        fh.write(f"-- Skipped:            {skipped}\n")
-        fh.write("--\n")
-        fh.write("-- WARNING: harness_events sin PK natural; aplicar 2 veces duplica filas.\n")
-        if not fragment:
-            fh.write("BEGIN TRANSACTION;\n")
-
-        for i in range(0, len(rows), BATCH_SIZE):
-            batch = rows[i : i + BATCH_SIZE]
-            fh.write(insert_prefix)
-            fh.write(",\n".join(row_values_sql(r) for r in batch))
-            fh.write(";\n")
-
-        if not fragment:
-            fh.write("COMMIT;\n")
-
-    print(f"[migrate_04:{project}] wrote {out} ({len(rows)} rows, {skipped} skipped)")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/tools/migration/migrate_04_harness_events.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# migrate_04_harness_events.sh
-# Wrapper: regenera el .sql desde events.jsonl y lo carga en ~/.gaia/gaia.db.
-#
-# OJO: harness_events no tiene PK natural. Re-ejecutar este wrapper duplica
-# filas. Si necesitas re-ejecutar limpio, primero elimina las filas con:
-#   sqlite3 ~/.gaia/gaia.db "DELETE FROM harness_events WHERE project='me';"
-set -euo pipefail
-
-HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PY_SCRIPT="${HERE}/migrate_04_harness_events.py"
-SQL_FILE="/tmp/migrate_04_harness_events.sql"
-DB_PATH="${HOME}/.gaia/gaia.db"
-
-# Paso 1: regenerar el .sql.
-echo "[migrate_04] regenerando ${SQL_FILE} ..."
-python3 "${PY_SCRIPT}"
-
-# Paso 2: aplicar el SQL (interceptado por el hook).
-echo "[migrate_04] aplicando ${SQL_FILE} en ${DB_PATH} ..."
-sqlite3 "${DB_PATH}" < "${SQL_FILE}"
-
-echo "[migrate_04] OK"

package/tools/scan/merge.py

@@ -1,213 +0,0 @@
-"""
-Section-Aware Context Combining Logic
-
-Merges scanner sections in-memory following the merge rules from
-contracts/merge-behavior.md. No file I/O -- gaia.db is the sole
-persistence layer.
-
-  Rule 1: Scanner-owned sections -> full replace
-  Rule 2: Agent-enriched sections -> never touch
-  Rule 3: Mixed sections -> selective update at sub-key level
-  Rule 4: Unknown/user-custom sections -> preserve
-  Rule 5: Metadata -> always update
-
-Note: Backward-compatible sections (project_details, application_architecture,
-development_standards) were removed in v3. Consumers read v2 scanner sections
-directly (project_identity, stack, git, environment, infrastructure).
-
-Special handling for sub-section level ownership: the `environment` section
-is jointly owned by the `tools` scanner (tools, tool_preferences) and the
-`environment` scanner (os, runtimes, env_files). Each scanner replaces only
-its owned sub-keys without clobbering the other scanner's sub-keys.
-
-Contract: specs/002-gaia-scan/contracts/merge-behavior.md
-"""
-
-import copy
-import logging
-from typing import Any, Dict, Optional, Set
-
-from tools.context.deep_merge import deep_merge
-
-logger = logging.getLogger(__name__)
-
-# Sections fully owned by scanners -- replaced entirely on each scan (Rule 1)
-# Top-level sections only; sub-key ownership handled separately
-SCANNER_OWNED_TOP_LEVEL: Dict[str, str] = {
-    "project_identity": "stack",
-    "stack": "stack",
-    "git": "git",
-    "infrastructure": "infrastructure",
-    "orchestration": "orchestration",
-    # "environment" is NOT listed here because it has sub-key ownership
-}
-
-# Sub-key ownership within the `environment` section (Rule 4 / sub-section)
-# Maps environment sub-key -> owning scanner name
-ENVIRONMENT_SUBKEY_OWNERS: Dict[str, str] = {
-    "tools": "tools",
-    "tool_preferences": "tools",
-    "os": "environment",
-    "runtimes": "environment",
-    "env_files": "environment",
-}
-
-# Agent-enriched sections -- never modified by scanners (Rule 2)
-AGENT_ENRICHED_SECTIONS: frozenset = frozenset([
-    "operational_guidelines",
-    "cluster_details",
-    "infrastructure_topology",
-    "monitoring_observability",
-    "architecture_overview",
-    "gcp_services",
-    "workload_identity",
-])
-
-# Mixed sections with partial scanner ownership (Rule 3)
-# Maps section_name -> set of scanner-owned field names
-MIXED_SECTION_SCANNER_FIELDS: Dict[str, Set[str]] = {
-    "terraform_infrastructure": {"layout"},
-    "gitops_configuration": {"repository"},
-    "application_services": {"base_path", "services"},
-}
-
-
-def merge_context(
-    existing: Dict[str, Any],
-    scan_sections: Dict[str, Any],
-    section_owners: Dict[str, str],
-) -> Dict[str, Any]:
-    """Merge scanner results with existing project-context sections.
-
-    Applies the merge rules from contracts/merge-behavior.md to produce
-    the final merged sections dict.
-
-    Args:
-        existing: Current sections from project-context.json (may be empty).
-        scan_sections: Combined sections from all scanners.
-        section_owners: Mapping of section/sub-section name to scanner name,
-                        from ScannerRegistry.get_section_owners().
-
-    Returns:
-        Merged sections dict ready to be written to project-context.json.
-        The merge is deterministic: same inputs always produce the same output.
-    """
-    result = copy.deepcopy(existing)
-
-    # --- Rule 1: Scanner-owned top-level sections -> full replace ---
-    for section_name in SCANNER_OWNED_TOP_LEVEL:
-        if section_name in scan_sections:
-            result[section_name] = copy.deepcopy(scan_sections[section_name])
-
-    # --- Sub-section level ownership for `environment` ---
-    # Both the `tools` scanner and `environment` scanner contribute sub-keys
-    # to the `environment` section. Each scanner's sub-keys replace their owned
-    # portion without clobbering the other scanner's sub-keys.
-    _merge_environment_section(result, scan_sections)
-
-    # --- Rule 2: Agent-enriched sections -> never touch ---
-    # These are already in `result` from the deepcopy of `existing`.
-    # We explicitly do NOT overwrite them, even if a scanner accidentally
-    # produced data for one of these section names.
-    # (No action needed -- they are preserved by the deepcopy.)
-
-    # --- Rule 3: Mixed sections -> selective update ---
-    for section_name, scanner_fields in MIXED_SECTION_SCANNER_FIELDS.items():
-        if section_name in scan_sections:
-            scan_data = scan_sections[section_name]
-            if section_name not in result:
-                result[section_name] = {}
-            # Only update scanner-owned fields; preserve agent fields
-            for field_name in scanner_fields:
-                if field_name in scan_data:
-                    result[section_name][field_name] = copy.deepcopy(
-                        scan_data[field_name]
-                    )
-
-    # --- Rule 5: Unknown/user-custom sections -> preserve ---
-    # Any section in `existing` that is not scanner-owned, not agent-enriched,
-    # not backward-compat, and not mixed is a user-custom section.
-    # These are already preserved by the initial deepcopy of `existing`.
-    # We do NOT add new unknown sections from scan_sections.
-
-    return result
-
-
-def _merge_environment_section(
-    result: Dict[str, Any],
-    scan_sections: Dict[str, Any],
-) -> None:
-    """Merge the `environment` section with sub-key level ownership.
-
-    Two scanners contribute to the `environment` section:
-    - `tools` scanner owns: tools, tool_preferences
-    - `environment` scanner owns: os, runtimes, env_files
-
-    Each scanner's sub-keys replace their owned portion; the other scanner's
-    sub-keys are preserved. The `_source` field gets a combined tag.
-
-    Args:
-        result: The result dict being built (mutated in place).
-        scan_sections: Combined sections from all scanners.
-    """
-    if "environment" not in scan_sections:
-        return
-
-    scan_env = scan_sections["environment"]
-
-    if "environment" not in result:
-        result["environment"] = {}
-
-    env = result["environment"]
-
-    # Replace each sub-key based on ownership
-    for subkey in ENVIRONMENT_SUBKEY_OWNERS:
-        if subkey in scan_env:
-            env[subkey] = copy.deepcopy(scan_env[subkey])
-
-    # Set combined _source tag
-    env["_source"] = "scanner:environment+tools"
-
-
-def collect_scanner_sections(
-    scanner_results: Dict[str, Any],
-) -> Dict[str, Any]:
-    """Collect and combine sections from all scanner results.
-
-    Handles the environment section specially: both `tools` and `environment`
-    scanners produce sub-keys under `environment`, so their outputs are
-    combined into a single `environment` section.
-
-    Args:
-        scanner_results: Mapping of scanner_name -> ScanResult (must have
-                         a `sections` attribute that is a dict).
-
-    Returns:
-        Combined sections dict from all scanners.
-    """
-    combined: Dict[str, Any] = {}
-    environment_parts: Dict[str, Any] = {}
-
-    for _scanner_name, scan_result in scanner_results.items():
-        sections = scan_result.sections if hasattr(scan_result, "sections") else {}
-
-        for section_name, section_data in sections.items():
-            if section_name == "environment":
-                # Merge environment sub-keys from both scanners
-                if isinstance(section_data, dict):
-                    for key, value in section_data.items():
-                        if key != "_source":
-                            environment_parts[key] = value
-            else:
-                # Non-environment sections: direct assignment (last scanner wins,
-                # but each section should have exactly one owner)
-                combined[section_name] = section_data
-
-    # Reassemble environment section if we got any parts
-    if environment_parts:
-        combined["environment"] = {
-            "_source": "scanner:environment+tools",
-            **environment_parts,
-        }
-
-    return combined