@lsctech/polaris 0.1.1 → 0.2.0

package/dist/workspace/workspace/.polaris/skills/caveman-compress/scripts/compress.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Caveman Memory Compression Orchestrator
+
+Usage:
+    python scripts/compress.py <filepath>
+"""
+
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+from typing import List
+
+OUTER_FENCE_REGEX = re.compile(
+    r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
+)
+
+# Filenames and paths that almost certainly hold secrets or PII. Compressing
+# them ships raw bytes to the Anthropic API — a third-party data boundary that
+# developers on sensitive codebases cannot cross. detect.py already skips .env
+# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
+# slip through the natural-language filter. This is a hard refuse before read.
+SENSITIVE_BASENAME_REGEX = re.compile(
+    r"(?ix)^("
+    r"\.env(\..+)?"
+    r"|\.netrc"
+    r"|credentials(\..+)?"
+    r"|secrets?(\..+)?"
+    r"|passwords?(\..+)?"
+    r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
+    r"|authorized_keys"
+    r"|known_hosts"
+    r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
+    r")$"
+)
+
+SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
+
+SENSITIVE_NAME_TOKENS = (
+    "secret", "credential", "password", "passwd",
+    "apikey", "accesskey", "token", "privatekey",
+)
+
+
+def is_sensitive_path(filepath: Path) -> bool:
+    """Heuristic denylist for files that must never be shipped to a third-party API."""
+    name = filepath.name
+    if SENSITIVE_BASENAME_REGEX.match(name):
+        return True
+    lowered_parts = {p.lower() for p in filepath.parts}
+    if lowered_parts & SENSITIVE_PATH_COMPONENTS:
+        return True
+    # Normalize separators so "api-key" and "api_key" both match "apikey".
+    lower = re.sub(r"[_\-\s.]", "", name.lower())
+    return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
+
+
+def strip_llm_wrapper(text: str) -> str:
+    """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
+    m = OUTER_FENCE_REGEX.match(text)
+    if m:
+        return m.group(2)
+    return text
+
+try:
+    from .detect import should_compress
+    from .validate import validate
+except ImportError:
+    import sys
+    import os
+    sys.path.insert(0, os.path.dirname(__file__))
+    from detect import should_compress  # type: ignore[no-redef]
+    from validate import validate  # type: ignore[no-redef]
+
+MAX_RETRIES = 2
+
+
+# ---------- Claude Calls ----------
+
+
+def call_claude(prompt: str) -> str:
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if api_key:
+        try:
+            import anthropic
+
+            client = anthropic.Anthropic(api_key=api_key)
+            msg = client.messages.create(
+                model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
+                max_tokens=8192,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return strip_llm_wrapper(msg.content[0].text.strip())
+        except ImportError:
+            pass  # anthropic not installed, fall back to CLI
+    # Fallback: use claude CLI (handles desktop auth)
+    claude_bin = os.environ.get("CLAUDE_BIN") or shutil.which("claude")
+    if claude_bin is None:
+        raise RuntimeError(
+            "Claude CLI not found. Set CLAUDE_BIN to the full path, "
+            "or ensure 'claude' is on your PATH."
+        )
+    print(f"Using Claude CLI: {claude_bin}")
+    try:
+        result = subprocess.run(
+            [claude_bin, "--print"],
+            input=prompt,
+            text=True,
+            capture_output=True,
+            check=True,
+            timeout=300,
+        )
+        return strip_llm_wrapper(result.stdout.strip())
+    except subprocess.TimeoutExpired as e:
+        raise RuntimeError("Claude CLI call timed out after 300 seconds.") from e
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Claude call failed:\n{e.stderr}") from e
+
+
+def build_compress_prompt(original: str) -> str:
+    return f"""
+Compress this markdown into caveman format.
+
+STRICT RULES:
+- Do NOT modify anything inside ``` code blocks
+- Do NOT modify anything inside inline backticks
+- Preserve ALL URLs exactly
+- Preserve ALL headings exactly
+- Preserve file paths and commands
+- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
+
+Only compress natural language.
+
+TEXT:
+{original}
+"""
+
+
+def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
+    errors_str = "\n".join(f"- {e}" for e in errors)
+    return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
+
+CRITICAL RULES:
+- DO NOT recompress or rephrase the file
+- ONLY fix the listed errors — leave everything else exactly as-is
+- The ORIGINAL is provided as reference only (to restore missing content)
+- Preserve caveman style in all untouched sections
+
+ERRORS TO FIX:
+{errors_str}
+
+HOW TO FIX:
+- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
+- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
+- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
+- Do not touch any section not mentioned in the errors
+
+ORIGINAL (reference only):
+{original}
+
+COMPRESSED (fix this):
+{compressed}
+
+Return ONLY the fixed compressed file. No explanation.
+"""
+
+
+# ---------- Core Logic ----------
+
+
+def compress_file(filepath: Path) -> bool:
+    # Resolve and validate path
+    filepath = filepath.resolve()
+    MAX_FILE_SIZE = 500_000  # 500KB
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {filepath}")
+    if filepath.stat().st_size > MAX_FILE_SIZE:
+        raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
+
+    # Refuse files that look like they contain secrets or PII. Compressing ships
+    # the raw bytes to the Anthropic API — a third-party boundary — so we fail
+    # loudly rather than silently exfiltrate credentials or keys. Override is
+    # intentional: the user must rename the file if the heuristic is wrong.
+    if is_sensitive_path(filepath):
+        raise ValueError(
+            f"Refusing to compress {filepath}: filename looks sensitive "
+            "(credentials, keys, secrets, or known private paths). "
+            "Compression sends file contents to the Anthropic API. "
+            "Rename the file if this is a false positive."
+        )
+
+    print(f"Processing: {filepath}")
+
+    if not should_compress(filepath):
+        print("Skipping (not natural language)")
+        return False
+
+    original_text = filepath.read_text(errors="replace")
+    backup_path = filepath.with_name(filepath.stem + ".original.md")
+
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
+    # Check if backup already exists to prevent accidental overwriting
+    if backup_path.exists():
+        print(f"⚠️ Backup file already exists: {backup_path}")
+        print("The original backup may contain important content.")
+        print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
+        return False
+
+    # Step 1: Compress
+    print("Compressing with Claude...")
+    compressed = call_claude(build_compress_prompt(original_text))
+
+    if compressed is None or not compressed.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    if compressed.strip() == original_text.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
+    backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="replace")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
+    filepath.write_text(compressed)
+
+    # Step 2: Validate + Retry
+    for attempt in range(MAX_RETRIES):
+        print(f"\nValidation attempt {attempt + 1}")
+
+        result = validate(backup_path, filepath)
+
+        if result.is_valid:
+            print("Validation passed")
+            break
+
+        print("❌ Validation failed:")
+        for err in result.errors:
+            print(f"   - {err}")
+
+        if attempt == MAX_RETRIES - 1:
+            # Restore original on failure
+            filepath.write_text(original_text)
+            backup_path.unlink(missing_ok=True)
+            print("❌ Failed after retries — original restored")
+            return False
+
+        print("Fixing with Claude...")
+        compressed = call_claude(
+            build_fix_prompt(original_text, compressed, result.errors)
+        )
+        filepath.write_text(compressed)
+
+    return True

package/dist/workspace/workspace/.polaris/skills/closeout-librarian/POLARIS.md

@@ -0,0 +1,46 @@
+# .polaris/skills/closeout-librarian/
+
+## Purpose
+
+This directory contains the Closeout Librarian skill package.
+
+The Closeout Librarian is a bounded runtime role dispatched by the Foreman exactly once
+per completed cluster, after all children are done and before PR creation.
+
+## Contents
+
+| File | Purpose |
+|------|---------|
+| `SKILL.md` | Skill entry point — read first, contains bootloader instructions and packet schema reference |
+| `chain.md` | Step-order execution map — the Librarian executes this in strict step order |
+| `steps/01-load-cluster-context.md` | Build work inventory from packet and cluster evidence |
+| `steps/02-reconcile-polaris-md.md` | Update affected POLARIS.md files |
+| `steps/03-reconcile-summary-md.md` | Refresh SUMMARY.md as continuation artifact |
+| `steps/04-doc-ingestion.md` | Ingest/promote/archive documentation |
+| `steps/05-link-validation.md` | Validate and repair broken links |
+| `steps/06-yaml-linking.md` | Update YAML references for promoted documents |
+| `steps/07-librarian-commit.md` | Commit documentation changes |
+| `steps/08-sealed-result.md` | Write sealed result JSON and terminate |
+
+## Invocation
+
+This skill is NOT invoked by user command. It is dispatched by the Foreman during
+step 08 of the `polaris-run` chain after cluster-complete is confirmed.
+
+The Foreman generates the packet via:
+```bash
+npm run polaris -- librarian packet <cluster-id>
+```
+
+Then dispatches the Librarian as a bounded subagent session.
+
+## Authority
+
+The Closeout Librarian may update POLARIS.md, SUMMARY.md, ingest documents, and commit
+documentation changes. It may NOT modify source code, runtime state, or create PRs.
+
+Full authority boundaries: `.polaris/roles/closeout-librarian.md`
+
+## Spec Reference
+
+`smartdocs/specs/active/closeout-librarian-spec.md`

package/dist/workspace/workspace/.polaris/skills/closeout-librarian/SKILL.md

@@ -0,0 +1,59 @@
+---
+name: closeout-librarian
+description: Reconcile completed cluster work into project cognition and documentation. Runs exactly once per cluster, after all children complete and before PR creation.
+role: closeout-librarian
+role_file: .polaris/roles/closeout-librarian.md
+---
+
+## Entry Points
+
+This skill is invoked by the Foreman as a dispatched session. It is NOT a user-facing command.
+
+Invocation: The Foreman dispatches the Closeout Librarian after confirming `cluster-complete` status
+and before running `polaris finalize`. The packet path is passed as the session prompt.
+
+---
+
+## Polaris Skill Bootloader
+
+This skill does NOT use the standard Polaris skill bootloader. The Closeout Librarian
+receives its packet directly from the Foreman — the packet is the sole authoritative
+instruction source for this session.
+
+The Librarian MUST:
+1. Read the packet at the path provided in the dispatch prompt.
+2. Validate the packet schema (role must be `"closeout-librarian"`).
+3. Execute `chain.md` in strict step order.
+4. Write the sealed result to `packet.result_path` before the session ends.
+
+If the packet cannot be read or fails schema validation, write a failure result immediately
+and terminate. Do not attempt to reconstruct the packet from context.
+
+---
+
+## Authority Reminder
+
+The Closeout Librarian may NOT modify implementation code, runtime state, or dispatch
+any workers. Its sole output is documentation/cognition changes + sealed result JSON.
+
+Full authority boundaries: `.polaris/roles/closeout-librarian.md`
+
+---
+
+## Packet Schema
+
+The packet is a `CloseoutLibrarianPacket`. See:
+- `smartdocs/specs/active/closeout-librarian-spec.md` §3 for full schema
+- `src/cognition/closeout-librarian-types.ts` for TypeScript types
+
+Required packet fields: `role`, `run_id`, `dispatch_id`, `cluster_id`,
+`completed_children`, `child_summaries`, `affected_folders`, `polaris_md_paths`,
+`result_path`, `prohibited_write_paths`, `allowed_write_paths`.
+
+---
+
+## Execution
+
+Read `chain.md` and execute steps in strict order.
+
+Do not skip steps. Do not reorder steps.

package/dist/workspace/workspace/.polaris/skills/closeout-librarian/chain.md

@@ -0,0 +1,116 @@
+---
+name: closeout-librarian-chain
+description: Step-order execution map for the Closeout Librarian. Runs once per completed cluster.
+---
+
+# Closeout Librarian Chain
+
+## Authority
+
+**The packet is authoritative. Do not infer cluster scope from conversation context.**
+
+Read the packet before any analysis. Execute steps in strict order.
+
+## Scope Contract
+
+The Closeout Librarian operates on documentation and cognition only.
+
+**Allowed writes:** POLARIS.md, SUMMARY.md (affected folders), smartdocs ingestion targets,
+cognition archive, YAML frontmatter in doc files, sealed result JSON, librarian commit.
+
+**Prohibited writes:** Source code, runtime state files, telemetry JSONL, cluster plan,
+Linear issue status, PR creation.
+
+---
+
+## Step Traversal Order
+
+```text
+01-load-cluster-context       ← Read packet, load cluster evidence, build work inventory
+02-drift-reconciliation       ← Run formal drift reconciliation checklist
+03-reconcile-polaris-md       ← Update affected POLARIS.md files to reflect current reality
+04-reconcile-summary-md       ← Refresh SUMMARY.md as continuation artifact
+05-doc-ingestion              ← Ingest/promote/archive documentation from completed work
+06-link-validation            ← Validate and repair broken links across affected docs
+07-yaml-linking               ← Update YAML references for ingested/promoted documents
+08-librarian-commit           ← Commit all documentation changes as sealed librarian commit
+09-sealed-result              ← Write CloseoutLibrarianResult JSON to result_path, terminate
+```
+
+All steps must complete in order. Termination after step 09 is mandatory.
+
+---
+
+## Execution Rules
+
+### Step Gate Behavior
+
+- Each step must complete before the next begins.
+- If a step encounters a recoverable error (e.g., one broken link that cannot be repaired),
+  record the blocker and continue.
+- If a step encounters a fatal error (e.g., packet unreadable, commit fails), record
+  `status: "failure"` in the result and proceed to step 09 (sealed result write).
+- Do not skip to step 09 for recoverable errors — complete the remaining steps.
+
+### Write Discipline
+
+Before writing any file, verify it is in `packet.allowed_write_paths`.
+If the target file is in `packet.prohibited_write_paths`, skip and record as blocker.
+
+### Commit Timing
+
+All writes occur during steps 03–07. No writes occur during step 08 or after.
+Step 08 creates exactly one git commit containing all documentation changes from this session.
+If no documentation changed (all steps found everything current), step 08 makes no commit.
+The result records `commit_sha: null` in this case — do not create an empty commit.
+
+### Result Timing
+
+Step 09 writes the sealed result JSON and terminates. No further writes are permitted after
+the sealed result is written.
+
+---
+
+## Narration Rules
+
+The Closeout Librarian does NOT produce user-facing narration during normal execution.
+
+**Allowed output:**
+- The sealed result JSON written to `result_path` (step 08)
+- Escalation output if blocked on a decision that requires operator input
+
+**Prohibited output:**
+- Step-by-step progress narration
+- Summary of what was found
+- Interim status updates
+
+All communication with the Foreman occurs through the sealed result and the commit record.
+
+---
+
+## Telemetry Events
+
+Emit the following structured events to the telemetry file (if path provided in packet):
+
+| Event | Step | Fields |
+|---|---|---|
+| `librarian-start` | 01 | `run_id`, `cluster_id`, `dispatch_id`, `timestamp` |
+| `librarian-step-complete` | each step | `step`, `run_id`, `timestamp`, `outcome` |
+| `librarian-commit` | 08 | `commit_sha`, `files_changed`, `timestamp` |
+| `librarian-complete` | 09 | `status`, `run_id`, `timestamp` |
+
+If no telemetry file is specified in the packet, skip telemetry events silently.
+
+---
+
+## Failure Handling
+
+| Condition | Action |
+|---|---|
+| Packet unreadable | Write failure result to `result_path` if possible, otherwise stderr. Terminate. |
+| Packet schema invalid | Write failure result, terminate |
+| POLARIS.md has unresolvable conflict | Record blocker, skip that file, continue |
+| Commit fails | Status: failure, record commit error, proceed to step 09 |
+| Result write fails | Stderr only. Cannot recover. |
+| All blockers, no writes | Status: `"blocked"`, list blockers, commit nothing |
+| Some work done, some blocked | Status: `"partial"`, describe both |

package/dist/workspace/workspace/.polaris/skills/closeout-librarian/steps/01-load-cluster-context.md

@@ -0,0 +1,78 @@
+---
+name: closeout-librarian-step-01
+description: Read packet, load cluster evidence, and build the work inventory for this reconciliation session.
+---
+
+# Step 01 — Load Cluster Context
+
+## Purpose
+
+Build a complete picture of what the cluster accomplished. This is the foundation for all
+subsequent reconciliation steps. Nothing is written in this step.
+
+## Actions
+
+### 1.1 Read and Validate Packet
+
+Read the packet from the path provided in the dispatch prompt.
+
+Validate required fields:
+- `role` must be `"closeout-librarian"`
+- `run_id`, `dispatch_id`, `cluster_id` must be present and non-empty
+- `completed_children` must be a non-empty array
+- `result_path` must be present
+- `allowed_write_paths` must be present
+
+If validation fails: write a minimal failure result to `result_path` and terminate.
+
+### 1.2 Load Child Summaries
+
+For each entry in `packet.child_summaries`:
+1. Read the compact-return result at `child_summary.compact_return_path` (if present)
+2. Read the cognition note at `child_summary.cognition_note_path` (if present)
+3. Note `child_summary.changed_files` — the primary file inventory for this cluster
+
+Build a combined `all_changed_files` list (deduplicated) across all children.
+
+### 1.3 Load Folder Cognition Context
+
+For each entry in `packet.polaris_md_paths`:
+1. Read `polaris_md` (current POLARIS.md for this folder)
+2. Read `summary_md` (current SUMMARY.md, if path provided and file exists)
+3. Read `cognition_index` (archive index, if path provided and file exists)
+
+Note which folders have pending cognition notes (from `packet.cognition_notes`).
+
+### 1.4 Build Work Inventory
+
+Produce an internal inventory (not written to disk):
+
+```yaml
+work_inventory:
+  cluster_id: <from packet>
+  completed_children: [<ids>]
+  all_changed_files: [<deduplicated list>]
+  affected_folders: [<from packet.affected_folders>]
+  pending_cognition_notes: [<paths>]
+  docs_candidate_for_ingestion: [<from packet.smartdocs_raw_paths>]
+  polaris_md_files: { <folder>: <current content> }
+  summary_md_files: { <folder>: <current content or null> }
+```
+
+### 1.5 Load Run Report
+
+Read the run report at `packet.run_report_path` (if present). This provides a structured
+summary of what work was done.
+
+### 1.6 Emit Telemetry
+
+Emit `librarian-start` event (if telemetry path provided).
+
+## Completion Criteria
+
+- Packet validated
+- All available child summaries loaded
+- All available folder cognition loaded
+- Work inventory built
+
+Proceed to step 02.