@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/core/reg-parser/scripts/preprocess-mandate.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+# preprocess-mandate.sh — Convert a mandate/regulatory document to structured Markdown
+# Usage: bash preprocess-mandate.sh <input-file> [--source "Visa VARS"] [--out /path/to/output.md] [--force]
+#
+# Refuses to overwrite a finished (non-RAW) mandate unless --force is given.
+#
+# Input:  PDF, DOCX, HTML, TXT, or Markdown
+# Output: Structured Markdown extract saved to references/mandates/<slug>.md
+#         (or custom path via --out)
+
+set -euo pipefail
+
+INPUT=""
+SOURCE_NAME=""
+OUTPUT=""
+FORCE=0
+
+# Parse args
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --source) SOURCE_NAME="$2"; shift 2 ;;
+    --out)    OUTPUT="$2"; shift 2 ;;
+    --force)  FORCE=1; shift ;;
+    -*)       echo "Unknown arg: $1" >&2; exit 1 ;;
+    *)        INPUT="$1"; shift ;;
+  esac
+done
+
+if [ -z "$INPUT" ]; then
+  echo "Usage: $0 <input-file> [--source 'Source Name'] [--out output.md]" >&2
+  exit 1
+fi
+
+if [ ! -f "$INPUT" ]; then
+  echo "❌ File not found: $INPUT" >&2
+  exit 1
+fi
+
+# Derive source name from filename if not provided
+if [ -z "$SOURCE_NAME" ]; then
+  BASENAME=$(basename "$INPUT")
+  SOURCE_NAME="${BASENAME%.*}"
+  echo "ℹ️  No --source provided, using filename: $SOURCE_NAME"
+fi
+
+# Derive slug for output filename
+SLUG=$(echo "$SOURCE_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//')
+DATE=$(date +%Y%m%d)
+
+# Determine output path
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MANDATES_DIR="$(dirname "$SCRIPT_DIR")/references/mandates"
+mkdir -p "$MANDATES_DIR"
+
+if [ -z "$OUTPUT" ]; then
+  # Check if a non-dated version already exists (known mandate)
+  if [ -f "$MANDATES_DIR/${SLUG}.md" ]; then
+    OUTPUT="$MANDATES_DIR/${SLUG}.md"
+    echo "ℹ️  Updating existing mandate: $OUTPUT"
+  else
+    # New ad-hoc mandate — use dated filename
+    OUTPUT="$MANDATES_DIR/adhoc-${SLUG}-${DATE}.md"
+  fi
+fi
+
+# A finished extract has no RAW marker; overwriting it would discard curated work.
+if [ -f "$OUTPUT" ] && [ "$FORCE" -ne 1 ] && ! grep -q 'Status:\*\* RAW' "$OUTPUT"; then
+  echo "❌ $OUTPUT is a finished mandate extract; refusing to overwrite it." >&2
+  echo "   Write to a new path (e.g. --out .../adhoc-${SLUG}-${DATE}.md) or pass --force." >&2
+  exit 1
+fi
+
+EXT="${INPUT##*.}"
+EXT_LOWER=$(echo "$EXT" | tr '[:upper:]' '[:lower:]')
+
+# Step 1: Convert to raw text
+RAW_TMP=$(mktemp /tmp/mandate-raw-XXXXXX.txt)
+trap "rm -f $RAW_TMP" EXIT
+
+echo "📄 Converting $INPUT → raw text..."
+
+case "$EXT_LOWER" in
+  pdf)
+    if command -v pdftotext &>/dev/null; then
+      pdftotext -layout "$INPUT" "$RAW_TMP"
+    elif command -v mutool &>/dev/null; then
+      mutool convert -F text -o "$RAW_TMP" "$INPUT"
+    elif python3 -c "import pdfminer" &>/dev/null 2>&1; then
+      python3 - "$INPUT" "$RAW_TMP" <<'PY'
+import sys
+from pdfminer.high_level import extract_text
+open(sys.argv[2], 'w').write(extract_text(sys.argv[1]))
+PY
+    else
+      echo "❌ No PDF converter found. Install: sudo apt install poppler-utils" >&2
+      exit 1
+    fi
+    ;;
+  docx|doc)
+    if command -v pandoc &>/dev/null; then
+      pandoc "$INPUT" -t plain -o "$RAW_TMP"
+    elif python3 -c "import docx2txt" &>/dev/null 2>&1; then
+      python3 - "$INPUT" "$RAW_TMP" <<'PY'
+import sys, docx2txt
+open(sys.argv[2], 'w').write(docx2txt.process(sys.argv[1]))
+PY
+    else
+      echo "❌ No DOCX converter. Install: sudo apt install pandoc" >&2
+      exit 1
+    fi
+    ;;
+  html|htm)
+    if command -v pandoc &>/dev/null; then
+      pandoc "$INPUT" -t plain -o "$RAW_TMP"
+    else
+      python3 - "$INPUT" "$RAW_TMP" <<'PY'
+import sys
+from html.parser import HTMLParser
+class P(HTMLParser):
+    def __init__(self): super().__init__(); self.t=[]
+    def handle_data(self,d): self.t.append(d)
+p=P(); p.feed(open(sys.argv[1]).read())
+open(sys.argv[2],'w').write('\n'.join(p.t))
+PY
+    fi
+    ;;
+  txt|md)
+    cp "$INPUT" "$RAW_TMP"
+    ;;
+  *)
+    echo "❌ Unsupported format: .$EXT_LOWER (supported: pdf, docx, doc, html, txt, md)" >&2
+    exit 1
+    ;;
+esac
+
+CHARS=$(wc -c < "$RAW_TMP")
+echo "✅ Converted: $CHARS chars"
+
+# Step 2: Write structured header for agent to complete
+# The agent reads the raw text and produces the structured extract.
+# This script writes the raw text + extraction template so the agent has both.
+
+cat > "$OUTPUT" << TEMPLATE
+# ${SOURCE_NAME} — Contract Obligation Extract
+
+> **Source file:** ${INPUT}
+> **Preprocessed:** ${DATE}
+> **Status:** RAW — agent extraction pending
+
+---
+
+## Raw Text (for agent extraction)
+
+~~~~~~~~
+$(cat "$RAW_TMP")
+~~~~~~~~
+
+---
+
+## Extraction Template
+
+<!-- Agent: replace this section with structured extract -->
+
+## Mandatory Downstream Agreement Clauses
+<!-- Clauses the counterparty MUST include in downstream/merchant agreements -->
+<!-- Format: **section [ref] [title]**: [obligation in one sentence] -->
+
+## Prohibited Conduct
+<!-- Explicitly prohibited actions or business types -->
+<!-- Format: **section [ref]**: [prohibition] -->
+
+## Thresholds & Triggers
+<!-- Numeric thresholds, ratios, timelines that apply to downstream entities -->
+<!-- Format: **section [ref] [metric]**: [value + consequence] -->
+
+## Audit & Oversight Rights
+<!-- Rights granted to regulator/scheme to enforce compliance -->
+<!-- Format: **section [ref]**: [right] -->
+
+## Notes
+<!-- Jurisdiction scope, effective date, version, caveats -->
+TEMPLATE
+
+echo ""
+echo "✅ Output: $OUTPUT"
+echo "   Next: agent reads this file and replaces the Extraction Template section"
+echo "   with structured content from the Raw Text."

package/skills/core/reg-parser/scripts/reg-parser.test.mjs

@@ -0,0 +1,165 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import test from 'node:test';
+
+const scriptsDir = dirname(fileURLToPath(import.meta.url));
+const skillDir = join(scriptsDir, '..');
+const checklistsDir = join(skillDir, 'references', 'checklists');
+const mandatesDir = join(skillDir, 'references', 'mandates');
+
+function run(cmd, args) {
+  return spawnSync(cmd, args, { encoding: 'utf8' });
+}
+
+function py(script, ...args) {
+  return run('python3', [join(scriptsDir, script), ...args]);
+}
+
+function scratch(name, content) {
+  const path = join(mkdtempSync(join(tmpdir(), 'reg-parser-test-')), name);
+  writeFileSync(path, content);
+  return path;
+}
+
+const HEADER = `\`\`\`yaml
+name: "Test Checklist"
+regulation: "Directive 2015/849"
+version: "1.0"
+scope: "Test"
+date: "2026-10-19"
+summary:
+  categories: 1
+  total_items: 1
+  critical: 1
+  major: 0
+  minor: 0
+\`\`\`
+
+## CAT-01: Customer Due Diligence
+
+`;
+
+// The item template in SKILL.md, filled in. If the template drifts from what
+// the validator accepts, every checklist an agent builds from it fails.
+function templateItem() {
+  const skill = readFileSync(join(skillDir, 'SKILL.md'), 'utf8');
+  const block = skill.match(/Each obligation from Step 2 becomes one checklist item:\n\n```markdown\n([\s\S]*?)```/)[1];
+  return block
+    .split('\n')
+    .filter((line) => !line.includes('# source-native scheme'))
+    .map((line) => line.replace(/\s+# cat scheme$/, ''))
+    .join('\n')
+    .replace('CAT-XX-NNN', 'CAT-01-001')
+    .replace('[Requirement title]', 'Verify identity')
+    .replace('[plain-language obligation]', 'Verify customer identity before onboarding.')
+    .replace('section [ref], [regulation]', 'Art. 13(1)(a), Directive 2015/849')
+    .replace('critical | major | minor', 'critical')
+    .replace('[Concrete observable step]', 'ID checked against a reliable source')
+    .replace('[Another step]', 'Check date recorded')
+    .replace('[Document or record that proves compliance]', 'Verification log');
+}
+
+test('every bundled checklist validates', () => {
+  for (const name of readdirSync(checklistsDir).filter((f) => f.endsWith('.md'))) {
+    const result = py('validate_checklist.py', join(checklistsDir, name));
+    assert.equal(result.status, 0, `${name}\n${result.stdout}${result.stderr}`);
+  }
+});
+
+test('every bundled mandate validates', () => {
+  for (const name of readdirSync(mandatesDir).filter((f) => f.endsWith('.md'))) {
+    const result = py('validate_mandate.py', join(mandatesDir, name));
+    assert.equal(result.status, 0, `${name}\n${result.stdout}${result.stderr}`);
+  }
+});
+
+test('every bundled checklist JSON equals a fresh dump of its markdown', () => {
+  for (const name of readdirSync(checklistsDir).filter((f) => f.endsWith('.json'))) {
+    const md = join(checklistsDir, name.replace(/\.json$/, '.md'));
+    const result = py('dump_checklist.py', md);
+    assert.equal(result.status, 0, result.stderr);
+    assert.deepEqual(JSON.parse(result.stdout), JSON.parse(readFileSync(join(checklistsDir, name), 'utf8')), name);
+  }
+});
+
+test('the SKILL.md item template passes the validator', () => {
+  const result = py('validate_checklist.py', scratch('ok.md', HEADER + templateItem() + '\n'));
+  assert.equal(result.status, 0, result.stdout + result.stderr);
+});
+
+test('a checklist with colons outside the bold labels fails', () => {
+  const item = templateItem().replace(/\*\*(\w+):\*\*/g, '**$1**:');
+  const result = py('validate_checklist.py', scratch('bad.md', HEADER + item + '\n'));
+  assert.equal(result.status, 1, result.stdout);
+});
+
+test('a checklist item with empty fields fails', () => {
+  const item = `### CAT-01-001: Empty
+
+- **Requirement:**
+- **Source:**
+- **Severity:** critical
+- **Verification:**
+- **Evidence:**
+`;
+  const result = py('validate_checklist.py', scratch('empty.md', HEADER + item));
+  assert.equal(result.status, 1, result.stdout);
+  assert.match(result.stdout + result.stderr, /field Requirement is empty/);
+  assert.match(result.stdout + result.stderr, /Verification has no '- \[ \]' steps/);
+});
+
+test('a raw preprocess output fails mandate validation', () => {
+  const source = scratch('src.md', 'Section 1 The licensee must keep records.\n```\nfenced\n```\n');
+  const out = join(dirname(source), 'mandate.md');
+  const pre = run('bash', [join(scriptsDir, 'preprocess-mandate.sh'), source, '--source', 'Test', '--out', out]);
+  assert.equal(pre.status, 0, pre.stderr);
+  const result = py('validate_mandate.py', out);
+  assert.equal(result.status, 1, result.stdout);
+  assert.match(result.stdout + result.stderr, /Status: RAW/);
+});
+
+test('preprocess refuses to overwrite a finished mandate', () => {
+  const finished = join(mandatesDir, 'visa-vars.md');
+  const copy = scratch('visa-vars.md', readFileSync(finished, 'utf8'));
+  const source = scratch('src.md', 'Section 1 must.\n');
+  const pre = run('bash', [join(scriptsDir, 'preprocess-mandate.sh'), source, '--source', 'Test', '--out', copy]);
+  assert.equal(pre.status, 1, pre.stdout);
+  assert.equal(readFileSync(copy, 'utf8'), readFileSync(finished, 'utf8'));
+});
+
+test('a mandate section with only prose or a hollow entry fails', () => {
+  const sections = (body) => `# Test — Contract Obligation Extract
+
+> **Source:** Test Act 2026
+
+## Mandatory Downstream Agreement Clauses
+${body}
+
+## Prohibited Conduct
+_None_
+
+## Thresholds & Triggers
+_None_
+
+## Audit & Oversight Rights
+_None_
+
+## Notes
+Jurisdiction: test only. Effective 2026-10-19. Version 1. Caveat: synthetic fixture for the validator.
+`;
+  assert.equal(py('validate_mandate.py', scratch('ok.md', sections('**section 4(1) Records**: Keep records five years.'))).status, 0);
+  assert.equal(py('validate_mandate.py', scratch('prose.md', sections('This part has no obligations worth noting, n/a mostly.'))).status, 1);
+  assert.equal(py('validate_mandate.py', scratch('hollow.md', sections('**section **:'))).status, 1);
+});
+
+test('compare_checklist counts item headings, not ID mentions', () => {
+  const target = join(skillDir, 'references', 'tests', 'vars-checklist.md');
+  const canonical = py('compare_checklist.py', join(checklistsDir, 'visa-vars-checklist.md'), target);
+  assert.equal(canonical.status, 0, canonical.stdout);
+  const ids = [...new Set(readFileSync(target, 'utf8').match(/\b[A-Z]{4}\.C\d+\.\d+\b/g))].join(' ');
+  assert.equal(py('compare_checklist.py', scratch('fake.md', ids), target).status, 1);
+});

package/skills/core/reg-parser/scripts/test_quarantine.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# test_quarantine.sh — Hide canonical reference checklists/mandates for a
+# given source slug while a test run extracts the same source from scratch.
+#
+# Without this, an agent running reg-parser against tests/vars-source.pdf
+# will discover references/checklists/visa-vars-checklist.md (listed in
+# SKILL.md as a built-in reference) and short-circuit by copying it,
+# defeating the test.
+#
+# Usage:
+#   bash test_quarantine.sh setup    <source-slug>
+#   bash test_quarantine.sh teardown <source-slug>
+#   bash test_quarantine.sh status   <source-slug>
+#
+# Example:
+#   bash test_quarantine.sh setup visa-vars
+#   # ... run agent against tests/vars-source.pdf ...
+#   bash test_quarantine.sh teardown visa-vars
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(dirname "$SCRIPT_DIR")"
+# Outside the skill tree (an agent searching it must not find the canonicals)
+# and outside /tmp (cleared on reboot, which would lose them mid-test).
+QUARANTINE_DIR="${REG_PARSER_QUARANTINE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/reg-parser-test-quarantine}"
+
+CMD="${1:-}"
+SLUG="${2:-}"
+
+if [ -z "$CMD" ] || [ -z "$SLUG" ]; then
+  echo "Usage: $0 setup|teardown|status <source-slug>" >&2
+  exit 2
+fi
+
+# Files that match a slug and should be hidden during the test.
+# Both checklists and mandates: an agent might consult either.
+TARGETS=(
+  "$SKILL_DIR/references/checklists/${SLUG}-checklist.md"
+  "$SKILL_DIR/references/checklists/${SLUG}-checklist.json"
+  "$SKILL_DIR/references/mandates/${SLUG}.md"
+)
+
+mkdir -p "$QUARANTINE_DIR/$SLUG"
+
+case "$CMD" in
+  setup)
+    moved=0
+    for t in "${TARGETS[@]}"; do
+      if [ -f "$t" ]; then
+        dest="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+        if [ -f "$dest" ]; then
+          echo "❌ Quarantine slot already taken: $dest" >&2
+          echo "   A previous setup didn't tear down. Run teardown first." >&2
+          exit 1
+        fi
+        mv "$t" "$dest"
+        echo "🔒 Hidden: $t -> $dest"
+        moved=$((moved + 1))
+      fi
+    done
+    if [ "$moved" -eq 0 ]; then
+      echo "ℹ️  No files matched slug '$SLUG' — nothing to quarantine."
+    fi
+    ;;
+
+  teardown)
+    restored=0
+    for t in "${TARGETS[@]}"; do
+      src="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+      if [ -f "$src" ]; then
+        if [ -f "$t" ]; then
+          echo "⚠️  Original path already populated, leaving quarantine intact: $t" >&2
+          echo "   (a fresh run wrote there; resolve manually)" >&2
+          continue
+        fi
+        mv "$src" "$t"
+        echo "🔓 Restored: $t"
+        restored=$((restored + 1))
+      fi
+    done
+    # Clean up empty slug dir
+    rmdir "$QUARANTINE_DIR/$SLUG" 2>/dev/null || true
+    if [ "$restored" -eq 0 ]; then
+      echo "ℹ️  Nothing in quarantine for slug '$SLUG'."
+    fi
+    ;;
+
+  status)
+    echo "Slug: $SLUG"
+    echo "Quarantine dir: $QUARANTINE_DIR/$SLUG"
+    for t in "${TARGETS[@]}"; do
+      src="$QUARANTINE_DIR/$SLUG/$(basename "$t")"
+      orig_present="no"; quar_present="no"
+      [ -f "$t" ] && orig_present="yes"
+      [ -f "$src" ] && quar_present="yes"
+      printf "  %s  orig=%s  quarantined=%s\n" "$(basename "$t")" "$orig_present" "$quar_present"
+    done
+    ;;
+
+  *)
+    echo "Unknown command: $CMD (use setup|teardown|status)" >&2
+    exit 2
+    ;;
+esac