@orderful/droid 0.45.1 → 0.46.0

package/src/tools/pii/skills/pii/SKILL.md

@@ -0,0 +1,97 @@
+---
+name: pii
+description: "Detect and redact PII in text files and directories. Use when scanning files for sensitive data before sharing, or redacting emails, phone numbers, credit cards, SSNs, and other PII. User prompts like '/pii scan transcript.md' or '/pii redact meeting-notes/ --dry-run'."
+argument-hint: "[scan | redact] {file|dir} [--entities TYPES] [--output PATH] [--dry-run] [--mask]"
+allowed-tools: [Read, Glob, Grep, Bash, Task]
+---
+
+# PII Skill
+
+Detect and redact personally identifiable information using Microsoft Presidio — deterministic pattern matching + NER, fully offline after first run.
+
+## Usage
+
+```bash
+/pii scan transcript.md                          # Scan a file for PII
+/pii scan ./meeting-notes/                       # Scan a directory recursively
+/pii redact transcript.md                        # Redact PII in-place (writes to transcript-redacted.md)
+/pii redact transcript.md --output clean.md      # Redact to specific output file
+/pii redact transcript.md --dry-run              # Preview redactions without writing
+/pii redact transcript.md --entities PHONE_NUMBER,EMAIL_ADDRESS  # Redact specific types only
+/pii redact transcript.md --mask                 # Use *** instead of <ENTITY_TYPE> placeholders
+```
+
+## Procedure
+
+### Step 0: Ensure Presidio venv is ready
+
+Before any scan or redact operation, verify the venv exists:
+
+```bash
+droid exec pii presidio-init
+```
+
+On first run this takes ~2–3 min (downloads Presidio packages + spaCy en_core_web_lg model). Subsequent runs return immediately (`already_existed: true`).
+
+If init fails, surface the error and stop — do not attempt to run analysis without the venv.
+
+### Step 1: Route on subcommand
+
+Parse the first argument:
+- `scan` → proceed to Step 2
+- `redact` → proceed to Step 3
+- anything else → show usage and stop
+
+### Step 2: Scan (delegate to pii-scanner agent)
+
+The `pii-scanner` agent runs analysis in an isolated context so raw PII values never appear in the parent conversation.
+
+**For a single file:**
+Delegate to the `pii-scanner` agent with `file_path` set to the absolute path.
+
+**For a directory:**
+Use `Glob` to find text files matching: `**/*.{md,txt,ts,js,json,yaml,yml,csv,html,xml}`
+Delegate each file to `pii-scanner` individually and aggregate results.
+
+**Display results:**
+Show entity types, counts, and affected line numbers only — never print raw PII values.
+
+Example output:
+```
+transcript.md: 3 PII entities found
+  EMAIL_ADDRESS × 2  (lines 4, 12)
+  PHONE_NUMBER × 1   (line 8)
+```
+
+### Step 3: Redact (run presidio-redact.ts directly)
+
+```bash
+droid exec pii presidio-redact \
+  --file <path> \
+  [--output <path>] \
+  [--dry-run] \
+  [--entities <TYPES>] \
+  [--mask]
+```
+
+Parse the JSON result and display:
+- Number of entities found and redacted
+- Output file path (if written)
+- In `--dry-run` mode: show a brief diff preview (first 10 redacted lines), but do NOT echo the full `redacted_text` field by default
+
+**Never echo `redacted_text` in full to the terminal** unless the user explicitly requests it.
+
+### Step 4: Format and present results
+
+- Always show: entity type, count, affected lines
+- On success for redact: confirm output path and entity counts
+- On failure: surface the error from the script's `error` field with actionable next steps
+
+## Behaviour Rules
+
+- **Never print raw PII values** to the terminal — always show types + line numbers only
+- For `--dry-run` redact, show a preview diff (10 lines max) not the full document
+- If `python3` is not found, show a clear install message from the prerequisite check
+- Binary files and unknown extensions are skipped during directory scans
+- The `pii-scanner` agent handles the isolation boundary for `scan`; `redact` operates directly since the redacted output is the goal
+- Supported entity types are listed in `references/supported-entities.md`

package/src/tools/pii/skills/pii/references/supported-entities.md

@@ -0,0 +1,90 @@
+# Supported PII Entity Types
+
+Reference for all entity types detectable by Microsoft Presidio (used by the `/pii` skill).
+
+Pass these names to `--entities` to filter detection:
+```bash
+/pii redact notes.md --entities EMAIL_ADDRESS,PHONE_NUMBER
+/pii scan transcript.md --entities US_SSN,CREDIT_CARD
+```
+
+---
+
+## Global Entities
+
+These entity types are supported across all languages and locales.
+
+| Entity | Description | Example | Detection Method |
+|--------|-------------|---------|-----------------|
+| `PERSON` | Full or partial personal names | John Smith, Jane | NER (spaCy) |
+| `EMAIL_ADDRESS` | Email addresses | user@example.com | Regex + validation |
+| `PHONE_NUMBER` | Phone numbers (various formats) | +1 555-123-4567, (555) 123-4567 | Regex |
+| `CREDIT_CARD` | Credit card numbers (Luhn-validated) | 4111 1111 1111 1111 | Regex + Luhn algorithm |
+| `IBAN_CODE` | International Bank Account Numbers | GB29 NWBK 6016 1331 9268 19 | Regex + checksum |
+| `IP_ADDRESS` | IPv4 and IPv6 addresses | 192.168.1.1, 2001:db8::1 | Regex |
+| `LOCATION` | Geographic locations and place names | New York, London, 123 Main St | NER (spaCy) |
+| `DATE_TIME` | Dates, times, and datetime values | 2024-01-15, January 15, 3:00 PM | NER (spaCy) |
+| `NRP` | Nationality, religion, political group | American, Christian | NER (spaCy) |
+| `MEDICAL_LICENSE` | Medical licence numbers | MD12345 | Regex |
+| `URL` | Web URLs | https://example.com/path | Regex |
+| `CRYPTO` | Cryptocurrency wallet addresses | 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2 | Regex |
+
+---
+
+## US Entities
+
+| Entity | Description | Example | Detection Method |
+|--------|-------------|---------|-----------------|
+| `US_SSN` | US Social Security Numbers | 123-45-6789 | Regex |
+| `US_PASSPORT` | US passport numbers | A12345678 | Regex |
+| `US_ITIN` | US Individual Taxpayer Identification Numbers | 912-00-0000 | Regex |
+| `US_DRIVER_LICENSE` | US driver's licence numbers (state-specific) | A1234567 | Regex |
+| `US_BANK_NUMBER` | US bank account numbers | 123456789012 | Regex |
+
+---
+
+## UK Entities
+
+| Entity | Description | Example | Detection Method |
+|--------|-------------|---------|-----------------|
+| `UK_NHS` | UK National Health Service numbers | 123 456 7890 | Regex + checksum |
+
+---
+
+## European Entities
+
+| Entity | Description | Example | Detection Method |
+|--------|-------------|---------|-----------------|
+| `ES_NIF` | Spanish NIF (tax ID) | 12345678A | Regex + checksum |
+| `IT_FISCAL_CODE` | Italian fiscal code | RSSMRA85T10A562S | Regex + checksum |
+| `IT_DRIVER_LICENSE` | Italian driver's licence | AA123456 | Regex |
+| `IT_VAT_CODE` | Italian VAT code | IT12345678901 | Regex |
+| `IT_PASSPORT` | Italian passport | AA1234567 | Regex |
+| `IT_IDENTITY_CARD` | Italian identity card | CA12345AA | Regex |
+| `PL_PESEL` | Polish PESEL national ID | 44051401458 | Regex + checksum |
+
+---
+
+## Asia-Pacific & Other Entities
+
+| Entity | Description | Example | Detection Method |
+|--------|-------------|---------|-----------------|
+| `SG_NRIC_FIN` | Singapore NRIC/FIN | S1234567A | Regex + checksum |
+| `AU_ABN` | Australian Business Number | 51 824 753 556 | Regex + checksum |
+| `AU_ACN` | Australian Company Number | 004 085 616 | Regex + checksum |
+| `AU_TFN` | Australian Tax File Number | 123 456 782 | Regex + checksum |
+| `AU_MEDICARE` | Australian Medicare number | 2123456701 | Regex + checksum |
+
+---
+
+## Notes
+
+- **NER-based** entities (PERSON, LOCATION, DATE_TIME, NRP) use the spaCy `en_core_web_sm` model. Accuracy depends on context — short, ambiguous names may be missed or misidentified.
+- **Regex + checksum** entities are highly accurate — a CREDIT_CARD match always passes Luhn's algorithm.
+- **Confidence scores** in `presidio-analyze.ts` output reflect detection certainty (0.0–1.0). Scores < 0.5 indicate uncertain matches.
+- For custom recognizers (Orderful-specific patterns like API keys, account IDs), see the v2 roadmap in issue #292.
+
+## References
+
+- [Presidio supported entities documentation](https://microsoft.github.io/presidio/supported_entities/)
+- [Adding custom recognizers](https://microsoft.github.io/presidio/analyzer/adding_recognizers/)

package/src/tools/pii/skills/pii/scripts/presidio-analyze.ts

@@ -0,0 +1,258 @@
+#!/usr/bin/env bun
+/**
+ * presidio-analyze
+ *
+ * Detect PII in a file or text string using Presidio.
+ * Shells out to the bundled Python venv.
+ *
+ * Usage:
+ *   bun run presidio-analyze.ts --file transcript.md
+ *   bun run presidio-analyze.ts --text "Call me at 555-1234"
+ *   bun run presidio-analyze.ts --file notes.md --entities EMAIL_ADDRESS,PHONE_NUMBER
+ *
+ * Output (JSON):
+ *   { "success": true, "entities": [{ "type": "EMAIL_ADDRESS", "start": 10, "end": 25, "score": 0.85, "line": 3 }] }
+ *   { "success": false, "error": "...", "init_required": true }
+ */
+
+import { execSync } from 'child_process';
+import { existsSync, mkdirSync, writeFileSync, unlinkSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+const VENV_PATH = join(process.env.HOME || '', '.droid', 'runtimes', 'presidio');
+const VENV_PYTHON = join(VENV_PATH, 'bin', 'python3');
+const MAX_BUFFER_BYTES = 50 * 1024 * 1024;
+const ENTITY_NAME_PATTERN = /^[A-Z0-9_]+$/;
+const SUPPORTED_ENTITIES = new Set([
+  'PERSON',
+  'EMAIL_ADDRESS',
+  'PHONE_NUMBER',
+  'CREDIT_CARD',
+  'IBAN_CODE',
+  'IP_ADDRESS',
+  'LOCATION',
+  'DATE_TIME',
+  'NRP',
+  'MEDICAL_LICENSE',
+  'URL',
+  'CRYPTO',
+  'US_SSN',
+  'US_PASSPORT',
+  'US_ITIN',
+  'US_DRIVER_LICENSE',
+  'US_BANK_NUMBER',
+  'UK_NHS',
+  'ES_NIF',
+  'IT_FISCAL_CODE',
+  'IT_DRIVER_LICENSE',
+  'IT_VAT_CODE',
+  'IT_PASSPORT',
+  'IT_IDENTITY_CARD',
+  'PL_PESEL',
+  'SG_NRIC_FIN',
+  'AU_ABN',
+  'AU_ACN',
+  'AU_TFN',
+  'AU_MEDICARE',
+]);
+
+interface Entity {
+  type: string;
+  start: number;
+  end: number;
+  score: number;
+  line: number;
+}
+
+interface AnalyzeResult {
+  success: boolean;
+  entities?: Entity[];
+  error?: string;
+  init_required?: boolean;
+}
+
+interface ParsedArgs {
+  file?: string;
+  text?: string;
+  entities?: string[];
+}
+
+function parseArgs(args: string[]): ParsedArgs {
+  const result: ParsedArgs = {};
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '--file' && args[i + 1]) {
+      result.file = args[++i];
+    } else if (arg === '--text' && args[i + 1]) {
+      result.text = args[++i];
+    } else if (arg === '--entities' && args[i + 1]) {
+      result.entities = args[++i].split(',').map(e => e.trim());
+    }
+  }
+
+  return result;
+}
+
+function computeLineNumber(text: string, offset: number): number {
+  const before = text.slice(0, offset);
+  return before.split('\n').length;
+}
+
+function validateEntities(entities: string[] | undefined): string | undefined {
+  if (!entities || entities.length === 0) {
+    return undefined;
+  }
+
+  for (const entity of entities) {
+    if (!ENTITY_NAME_PATTERN.test(entity)) {
+      return `Invalid entity type: ${entity}. Allowed pattern: ${ENTITY_NAME_PATTERN.source}`;
+    }
+
+    if (!SUPPORTED_ENTITIES.has(entity)) {
+      return `Unsupported entity type: ${entity}`;
+    }
+  }
+
+  return undefined;
+}
+
+function run(cmd: string): { ok: boolean; stdout: string; stderr: string } {
+  try {
+    const output = execSync(cmd, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+      maxBuffer: MAX_BUFFER_BYTES,
+    });
+    return { ok: true, stdout: output, stderr: '' };
+  } catch (err: unknown) {
+    const error = err as { stdout?: string; stderr?: string; message?: string };
+    return {
+      ok: false,
+      stdout: error.stdout || '',
+      stderr: error.stderr || error.message || 'Unknown error',
+    };
+  }
+}
+
+function presidioAnalyze(parsed: ParsedArgs): AnalyzeResult {
+  // Validate venv exists
+  if (!existsSync(VENV_PYTHON)) {
+    return {
+      success: false,
+      error: 'Presidio venv not found. Run presidio-init.ts first.',
+      init_required: true,
+    };
+  }
+
+  // Validate input
+  if (!parsed.file && !parsed.text) {
+    return {
+      success: false,
+      error: 'Either --file or --text is required.',
+    };
+  }
+
+  const entitiesError = validateEntities(parsed.entities);
+  if (entitiesError) {
+    return {
+      success: false,
+      error: entitiesError,
+    };
+  }
+
+  // Read source text for line number computation
+  let sourceText: string;
+  if (parsed.file) {
+    if (!existsSync(parsed.file)) {
+      return { success: false, error: `File not found: ${parsed.file}` };
+    }
+    try {
+      sourceText = readFileSync(parsed.file, 'utf-8');
+    } catch (err: unknown) {
+      const e = err as { message?: string };
+      return { success: false, error: `Failed to read file: ${e.message}` };
+    }
+  } else {
+    sourceText = parsed.text!;
+  }
+
+  // Build Python inline script
+  const entitiesArg = parsed.entities && parsed.entities.length > 0
+    ? `entities=[${parsed.entities.map(e => `"${e}"`).join(', ')}]`
+    : '';
+
+  const pythonScript = `
+import sys, json
+from presidio_analyzer import AnalyzerEngine
+
+engine = AnalyzerEngine()
+text = ${JSON.stringify(sourceText)}
+results = engine.analyze(text=text, language='en'${entitiesArg ? ', ' + entitiesArg : ''})
+output = []
+for r in results:
+    output.append({
+        'type': r.entity_type,
+        'start': r.start,
+        'end': r.end,
+        'score': round(r.score, 4)
+    })
+print(json.dumps(output))
+`.trim();
+
+  // Write tmp script
+  const tmpDir = tmpdir();
+  const tmpScript = join(tmpDir, `pii-analyze-${Date.now()}.py`);
+
+  try {
+    mkdirSync(tmpDir, { recursive: true });
+    writeFileSync(tmpScript, pythonScript, 'utf-8');
+  } catch (err: unknown) {
+    const e = err as { message?: string };
+    return { success: false, error: `Failed to write temp script: ${e.message}` };
+  }
+
+  try {
+    const result = run(`"${VENV_PYTHON}" "${tmpScript}"`);
+
+    if (!result.ok) {
+      return {
+        success: false,
+        error: `Presidio analysis failed: ${result.stderr}`,
+      };
+    }
+
+    let rawEntities: Array<{ type: string; start: number; end: number; score: number }>;
+    try {
+      rawEntities = JSON.parse(result.stdout.trim());
+    } catch {
+      return { success: false, error: `Failed to parse Presidio output: ${result.stdout}` };
+    }
+
+    // Compute line numbers
+    const entities: Entity[] = rawEntities.map(e => ({
+      ...e,
+      line: computeLineNumber(sourceText, e.start),
+    }));
+
+    return { success: true, entities };
+  } finally {
+    // Clean up tmp file
+    try {
+      unlinkSync(tmpScript);
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}
+
+// Main
+const args = process.argv.slice(2);
+const parsed = parseArgs(args);
+const result = presidioAnalyze(parsed);
+console.log(JSON.stringify(result, null, 2));
+
+if (!result.success) {
+  process.exit(1);
+}

package/src/tools/pii/skills/pii/scripts/presidio-init.ts

@@ -0,0 +1,151 @@
+#!/usr/bin/env bun
+/**
+ * presidio-init
+ *
+ * One-time bootstrap for the Presidio Python venv.
+ * Idempotent: no-op if already initialised (marker file check + venv binary check).
+ *
+ * Usage:
+ *   bun run presidio-init.ts
+ *
+ * Output (JSON):
+ *   { "success": true, "already_existed": true }
+ *   { "success": true, "initialized": true, "python_path": "...", "venv_path": "..." }
+ *   { "success": false, "error": "..." }
+ */
+
+import { execSync } from 'child_process';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+
+const VENV_PATH = join(process.env.HOME || '', '.droid', 'runtimes', 'presidio');
+const MARKER_FILE = join(VENV_PATH, '.droid-initialized');
+const VENV_PYTHON = join(VENV_PATH, 'bin', 'python3');
+
+interface InitResult {
+  success: boolean;
+  already_existed?: boolean;
+  initialized?: boolean;
+  python_path?: string;
+  venv_path?: string;
+  error?: string;
+}
+
+function run(
+  cmd: string,
+  opts: { visible?: boolean; cwd?: string } = {}
+): { ok: boolean; output: string } {
+  try {
+    const output = execSync(cmd, {
+      cwd: opts.cwd,
+      encoding: 'utf-8',
+      stdio: opts.visible ? 'inherit' : ['pipe', 'pipe', 'pipe'],
+    });
+    return { ok: true, output: typeof output === 'string' ? output.trim() : '' };
+  } catch (err: unknown) {
+    const error = err as { stderr?: string; message?: string };
+    return { ok: false, output: error.stderr || error.message || 'Unknown error' };
+  }
+}
+
+function checkPythonVersion(): { ok: boolean; version?: string; error?: string } {
+  const result = run('python3 --version');
+  if (!result.ok) {
+    return { ok: false, error: 'python3 not found. Install Python 3.8+ from python.org or via: brew install python3' };
+  }
+
+  const match = result.output.match(/Python (\d+)\.(\d+)/);
+  if (!match) {
+    return { ok: false, error: `Could not parse Python version from: ${result.output}` };
+  }
+
+  const major = parseInt(match[1], 10);
+  const minor = parseInt(match[2], 10);
+
+  if (major < 3 || (major === 3 && minor < 8)) {
+    return {
+      ok: false,
+      error: `Python 3.8+ required, found ${result.output.trim()}. Upgrade via: brew install python3`,
+    };
+  }
+
+  return { ok: true, version: result.output.trim() };
+}
+
+function presidioInit(): InitResult {
+  // Fast path: marker file + binary both exist
+  if (existsSync(MARKER_FILE) && existsSync(VENV_PYTHON)) {
+    return { success: true, already_existed: true };
+  }
+
+  // Check Python version before attempting setup
+  const pythonCheck = checkPythonVersion();
+  if (!pythonCheck.ok) {
+    return { success: false, error: pythonCheck.error };
+  }
+
+  // Create parent directory
+  try {
+    mkdirSync(join(process.env.HOME || '', '.droid', 'runtimes'), { recursive: true });
+  } catch (err: unknown) {
+    const e = err as { message?: string };
+    return { success: false, error: `Failed to create runtimes directory: ${e.message}` };
+  }
+
+  // Step 1: Create venv
+  console.error('[pii] Creating Python venv (first run — this takes ~2–3 min)...');
+  const venvResult = run(`python3 -m venv "${VENV_PATH}"`, { visible: true });
+  if (!venvResult.ok) {
+    return { success: false, error: `Failed to create venv: ${venvResult.output}` };
+  }
+
+  // Step 2: Upgrade pip
+  console.error('[pii] Upgrading pip...');
+  const pipUpgrade = run(`"${VENV_PYTHON}" -m pip install --quiet --upgrade pip`, { visible: true });
+  if (!pipUpgrade.ok) {
+    return { success: false, error: `Failed to upgrade pip: ${pipUpgrade.output}` };
+  }
+
+  // Step 3: Install Presidio packages
+  console.error('[pii] Installing presidio-analyzer, presidio-anonymizer, spacy...');
+  const installResult = run(
+    `"${VENV_PYTHON}" -m pip install --quiet presidio-analyzer presidio-anonymizer spacy`,
+    { visible: true }
+  );
+  if (!installResult.ok) {
+    return { success: false, error: `Failed to install Presidio: ${installResult.output}` };
+  }
+
+  // Step 4: Download spaCy model (~400 MB — used by Presidio's default NLP engine)
+  console.error('[pii] Downloading spaCy en_core_web_lg model (~400 MB)...');
+  const spaCyResult = run(
+    `"${VENV_PYTHON}" -m spacy download en_core_web_lg`,
+    { visible: true }
+  );
+  if (!spaCyResult.ok) {
+    return { success: false, error: `Failed to download spaCy model: ${spaCyResult.output}` };
+  }
+
+  // Write marker file
+  try {
+    writeFileSync(MARKER_FILE, new Date().toISOString());
+  } catch (err: unknown) {
+    const e = err as { message?: string };
+    return { success: false, error: `Failed to write marker file: ${e.message}` };
+  }
+
+  return {
+    success: true,
+    initialized: true,
+    python_path: VENV_PYTHON,
+    venv_path: VENV_PATH,
+  };
+}
+
+// Main
+const result = presidioInit();
+console.log(JSON.stringify(result, null, 2));
+
+if (!result.success) {
+  process.exit(1);
+}