@1aboveio/skills 0.20.2 → 0.20.4

package/skills/document-utils/reg-parser/scripts/preprocess-mandate.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+# preprocess-mandate.sh — Convert a mandate/regulatory document to structured Markdown
+# Usage: bash preprocess-mandate.sh <input-file> [--source "Visa VARS"] [--out /path/to/output.md]
+#
+# Input:  PDF, DOCX, HTML, TXT, or Markdown
+# Output: Structured Markdown extract saved to references/mandates/<slug>.md
+#         (or custom path via --out)
+
+set -euo pipefail
+
+INPUT=""
+SOURCE_NAME=""
+OUTPUT=""
+
+# Parse args
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --source) SOURCE_NAME="$2"; shift 2 ;;
+    --out)    OUTPUT="$2"; shift 2 ;;
+    -*)       echo "Unknown arg: $1" >&2; exit 1 ;;
+    *)        INPUT="$1"; shift ;;
+  esac
+done
+
+if [ -z "$INPUT" ]; then
+  echo "Usage: $0 <input-file> [--source 'Source Name'] [--out output.md]" >&2
+  exit 1
+fi
+
+if [ ! -f "$INPUT" ]; then
+  echo "❌ File not found: $INPUT" >&2
+  exit 1
+fi
+
+# Derive source name from filename if not provided
+if [ -z "$SOURCE_NAME" ]; then
+  BASENAME=$(basename "$INPUT")
+  SOURCE_NAME="${BASENAME%.*}"
+  echo "ℹ️  No --source provided, using filename: $SOURCE_NAME"
+fi
+
+# Derive slug for output filename
+SLUG=$(echo "$SOURCE_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+DATE=$(date +%Y%m%d)
+
+# Determine output path
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MANDATES_DIR="$(dirname "$SCRIPT_DIR")/references/mandates"
+mkdir -p "$MANDATES_DIR"
+
+if [ -z "$OUTPUT" ]; then
+  # Check if a non-dated version already exists (known mandate)
+  if [ -f "$MANDATES_DIR/${SLUG}.md" ]; then
+    OUTPUT="$MANDATES_DIR/${SLUG}.md"
+    echo "ℹ️  Updating existing mandate: $OUTPUT"
+  else
+    # New ad-hoc mandate — use dated filename
+    OUTPUT="$MANDATES_DIR/adhoc-${SLUG}-${DATE}.md"
+  fi
+fi
+
+EXT="${INPUT##*.}"
+EXT_LOWER=$(echo "$EXT" | tr '[:upper:]' '[:lower:]')
+
+# Step 1: Convert to raw text
+RAW_TMP=$(mktemp /tmp/mandate-raw-XXXXXX.txt)
+trap "rm -f $RAW_TMP" EXIT
+
+echo "📄 Converting $INPUT → raw text..."
+
+case "$EXT_LOWER" in
+  pdf)
+    if command -v pdftotext &>/dev/null; then
+      pdftotext -layout "$INPUT" "$RAW_TMP"
+    elif command -v mutool &>/dev/null; then
+      mutool convert -F text -o "$RAW_TMP" "$INPUT"
+    elif python3 -c "import pdfminer" &>/dev/null 2>&1; then
+      python3 -c "
+from pdfminer.high_level import extract_text
+open('$RAW_TMP', 'w').write(extract_text('$INPUT'))
+"
+    else
+      echo "❌ No PDF converter found. Install: sudo apt install poppler-utils" >&2
+      exit 1
+    fi
+    ;;
+  docx|doc)
+    if command -v pandoc &>/dev/null; then
+      pandoc "$INPUT" -t plain -o "$RAW_TMP"
+    elif python3 -c "import docx2txt" &>/dev/null 2>&1; then
+      python3 -c "import docx2txt; open('$RAW_TMP','w').write(docx2txt.process('$INPUT'))"
+    else
+      echo "❌ No DOCX converter. Install: sudo apt install pandoc" >&2
+      exit 1
+    fi
+    ;;
+  html|htm)
+    if command -v pandoc &>/dev/null; then
+      pandoc "$INPUT" -t plain -o "$RAW_TMP"
+    else
+      python3 -c "
+from html.parser import HTMLParser
+class P(HTMLParser):
+    def __init__(self): super().__init__(); self.t=[]
+    def handle_data(self,d): self.t.append(d)
+p=P(); p.feed(open('$INPUT').read())
+open('$RAW_TMP','w').write('\n'.join(p.t))
+"
+    fi
+    ;;
+  txt|md)
+    cp "$INPUT" "$RAW_TMP"
+    ;;
+  *)
+    echo "❌ Unsupported format: .$EXT_LOWER (supported: pdf, docx, doc, html, txt, md)" >&2
+    exit 1
+    ;;
+esac
+
+CHARS=$(wc -c < "$RAW_TMP")
+echo "✅ Converted: $CHARS chars"
+
+# Step 2: Write structured header for agent to complete
+# The agent reads the raw text and produces the structured extract.
+# This script writes the raw text + extraction template so the agent has both.
+
+cat > "$OUTPUT" << TEMPLATE
+# ${SOURCE_NAME} — Contract Obligation Extract
+
+> **Source file:** ${INPUT}
+> **Preprocessed:** ${DATE}
+> **Status:** RAW — agent extraction pending
+
+---
+
+## Raw Text (for agent extraction)
+
+\`\`\`
+$(cat "$RAW_TMP")
+\`\`\`
+
+---
+
+## Extraction Template
+
+<!-- Agent: replace this section with structured extract -->
+
+## Mandatory Downstream Agreement Clauses
+<!-- Clauses the counterparty MUST include in downstream/merchant agreements -->
+<!-- Format: **§[ref] [title]**: [obligation in one sentence] -->
+
+## Prohibited Conduct
+<!-- Explicitly prohibited actions or business types -->
+<!-- Format: **§[ref]**: [prohibition] -->
+
+## Thresholds & Triggers
+<!-- Numeric thresholds, ratios, timelines that apply to downstream entities -->
+<!-- Format: **§[ref] [metric]**: [value + consequence] -->
+
+## Audit & Oversight Rights
+<!-- Rights granted to regulator/scheme to enforce compliance -->
+<!-- Format: **§[ref]**: [right] -->
+
+## Notes
+<!-- Jurisdiction scope, effective date, version, caveats -->
+TEMPLATE
+
+echo ""
+echo "✅ Output: $OUTPUT"
+echo "   Next: agent reads this file and replaces the Extraction Template section"
+echo "   with structured content from the Raw Text."

package/skills/document-utils/reg-parser/scripts/test_quarantine.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# test_quarantine.sh — Hide canonical reference checklists/mandates for a
+# given source slug while a test run extracts the same source from scratch.
+#
+# Without this, an agent running reg-parser against tests/vars-source.pdf
+# will discover references/checklists/visa-vars-checklist.md (listed in
+# SKILL.md as a built-in reference) and short-circuit by copying it,
+# defeating the test.
+#
+# Usage:
+#   bash test_quarantine.sh setup    <source-slug>
+#   bash test_quarantine.sh teardown <source-slug>
+#   bash test_quarantine.sh status   <source-slug>
+#
+# Example:
+#   bash test_quarantine.sh setup visa-vars
+#   # ... run agent against tests/vars-source.pdf ...
+#   bash test_quarantine.sh teardown visa-vars
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(dirname "$SCRIPT_DIR")"
+QUARANTINE_DIR="/tmp/reg-parser-test-quarantine"
+
+CMD="${1:-}"
+SLUG="${2:-}"
+
+if [ -z "$CMD" ] || [ -z "$SLUG" ]; then
+  echo "Usage: $0 setup|teardown|status <source-slug>" >&2
+  exit 2
+fi
+
+# Files that match a slug and should be hidden during the test.
+# Both checklists and mandates: an agent might consult either.
+TARGETS=(
+  "$SKILL_DIR/references/checklists/${SLUG}-checklist.md"
+  "$SKILL_DIR/references/mandates/${SLUG}.md"
+)
+
+mkdir -p "$QUARANTINE_DIR/$SLUG"
+
+case "$CMD" in
+  setup)
+    moved=0
+    for t in "${TARGETS[@]}"; do
+      if [ -f "$t" ]; then
+        dest="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+        if [ -f "$dest" ]; then
+          echo "❌ Quarantine slot already taken: $dest" >&2
+          echo "   A previous setup didn't tear down. Run teardown first." >&2
+          exit 1
+        fi
+        mv "$t" "$dest"
+        echo "🔒 Hidden: $t -> $dest"
+        moved=$((moved + 1))
+      fi
+    done
+    if [ "$moved" -eq 0 ]; then
+      echo "ℹ️  No files matched slug '$SLUG' — nothing to quarantine."
+    fi
+    ;;
+
+  teardown)
+    restored=0
+    for t in "${TARGETS[@]}"; do
+      src="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+      if [ -f "$src" ]; then
+        if [ -f "$t" ]; then
+          echo "⚠️  Original path already populated, leaving quarantine intact: $t" >&2
+          echo "   (a fresh run wrote there; resolve manually)" >&2
+          continue
+        fi
+        mv "$src" "$t"
+        echo "🔓 Restored: $t"
+        restored=$((restored + 1))
+      fi
+    done
+    # Clean up empty slug dir
+    rmdir "$QUARANTINE_DIR/$SLUG" 2>/dev/null || true
+    if [ "$restored" -eq 0 ]; then
+      echo "ℹ️  Nothing in quarantine for slug '$SLUG'."
+    fi
+    ;;
+
+  status)
+    echo "Slug: $SLUG"
+    echo "Quarantine dir: $QUARANTINE_DIR/$SLUG"
+    for t in "${TARGETS[@]}"; do
+      src="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+      orig_present="no"; quar_present="no"
+      [ -f "$t" ] && orig_present="yes"
+      [ -f "$src" ] && quar_present="yes"
+      printf "  %s  orig=%s  quarantined=%s\n" "$(basename "$t")" "$orig_present" "$quar_present"
+    done
+    ;;
+
+  *)
+    echo "Unknown command: $CMD (use setup|teardown|status)" >&2
+    exit 2
+    ;;
+esac