pwn 0.5.631 → 0.5.632

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f125b1fd9398d2de99e6b311a1950e646992736f38552b818cb269e9d8704a6d
-  data.tar.gz: a84266bb38a93fecb6e8c5fa0b5c73b638dcaa9196838a436c31127f95eedf31
+  metadata.gz: 5bea724f1a504fa45e1f4eaa786761599a75e08a07b5848327b1dafd61dbe6f2
+  data.tar.gz: 9a4e693fe7b54d7477896d78272fb5c87a42deb5719730489a0a623eb39230dd
 SHA512:
-  metadata.gz: 9a2114b37283201211235e16fb558b51e5eb64ebec22d164898fc5b40db763696f56fdea8a99e9cf3905701fa3738a866726822a797e970a1de64c3ded54503d
-  data.tar.gz: fb3452718e38139450c34b8de7905188906e5d10935aabb592f6c96c0789e61ab1e29d4b4bc6e4563c1c124ffa87c835c4c417bccd99a0c919a93443e757f70f
+  metadata.gz: 3998a8a58340cdf1c8b4a9ba9a57a8fe9fb7a749117a606f65b643996ec343606692d5382f71ccd6e119ceac6f995047092bdd87f2e14e8e575b1a306ee29b1f
+  data.tar.gz: c10128b60319de27d0404c18289b91dd40ca78ca18df522e788c6b2a968e85c8b71c49c2447ef6eee268538787c1b09ba786a303a9fe8eb5198912eb4834ab90

data/.github/workflows/install-matrix.yml

@@ -0,0 +1,177 @@
+# .github/workflows/install-matrix.yml
+#
+# Layer 3 of the Installation.md verification strategy: prove the
+# fresh-host install path ("gem install pwn && pwn setup") on clean
+# images for every distro / package manager the doc claims to support,
+# then assert the result *via rspec* so failures read like the rest of
+# the test suite.
+#
+# Layers 1 & 2 (doc↔code parity, behavioural contracts) live in
+# spec/documentation/installation_md_spec.rb.
+# Layer 3's functional assertions live in
+# spec/integration/fresh_install_spec.rb and only activate when
+# PWN_FRESH_INSTALL is set — this workflow sets it, `rake` on a
+# developer box does not.
+#
+# This is world-drift, not repo-drift — distro package names change
+# upstream — so it also runs nightly.
+name: install-matrix
+
+on:
+  push:
+    paths:
+      - 'lib/pwn/setup.rb'
+      - 'bin/pwn_setup'
+      - 'bin/pwn'
+      - 'documentation/Installation.md'
+      - 'install.sh'
+      - 'packer/provisioners/pwn.sh'
+      - 'pwn.gemspec'
+      - '.github/workflows/install-matrix.yml'
+      - 'spec/documentation/installation_md_spec.rb'
+      - 'spec/integration/**'
+  pull_request:
+    paths:
+      - 'lib/pwn/setup.rb'
+      - 'bin/pwn_setup'
+      - 'bin/pwn'
+      - 'documentation/Installation.md'
+      - 'install.sh'
+      - 'packer/provisioners/pwn.sh'
+      - 'pwn.gemspec'
+      - 'spec/documentation/installation_md_spec.rb'
+      - 'spec/integration/**'
+  schedule:
+    - cron: '17 4 * * *'   # nightly — catch upstream distro package-name drift
+  workflow_dispatch:
+
+jobs:
+  linux:
+    name: ${{ matrix.image }} / ${{ matrix.pm }} / ${{ matrix.profile }}
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.image }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Base axes → 5 images × 2 profiles = 10 legs.
+        image:
+          - kalilinux/kali-rolling
+          - debian:bookworm
+          - ubuntu:24.04
+          - fedora:latest
+          - archlinux:latest
+        profile:
+          - core
+          - net
+        # `include` here EXTENDS each base combo whose `image` matches
+        # with the per-distro pm + ruby-bootstrap command; it does NOT
+        # add new combos. (Putting these under `include` alone with a
+        # separate top-level `profile:` would collapse to 2 legs — the
+        # bug this rewrite fixes.)
+        include:
+          - image: kalilinux/kali-rolling
+            pm: apt
+            ruby: apt-get update && apt-get install -y ruby-full ruby-dev build-essential git
+          - image: debian:bookworm
+            pm: apt
+            ruby: apt-get update && apt-get install -y ruby-full ruby-dev build-essential git
+          - image: ubuntu:24.04
+            pm: apt
+            ruby: apt-get update && apt-get install -y ruby-full ruby-dev build-essential git
+          - image: fedora:latest
+            pm: dnf
+            ruby: dnf install -y ruby ruby-devel redhat-rpm-config gcc gcc-c++ make git
+          - image: archlinux:latest
+            pm: pacman
+            ruby: pacman -Sy --noconfirm ruby base-devel git
+    env:
+      DEBIAN_FRONTEND: noninteractive
+      PWN_FRESH_INSTALL: '1'
+      PWN_EXPECTED_PM: ${{ matrix.pm }}
+      PWN_PROFILE: ${{ matrix.profile }}
+    steps:
+      - name: bootstrap ruby (${{ matrix.pm }})
+        run: ${{ matrix.ruby }}
+
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: build & install gem from checkout (git-checkout path in doc)
+        run: |
+          gem build pwn.gemspec
+          gem install --no-document ./pwn-*.gem
+          gem install --no-document rspec
+          ruby -r pwn -e 'puts "pwn #{PWN::VERSION} ruby #{RUBY_VERSION} #{RUBY_PLATFORM}"'
+
+      - name: dry-run — every package name resolves for this distro
+        run: pwn setup --profile full --dry-run --yes
+
+      - name: provision profile (documentation/Installation.md verbatim)
+        run: pwn setup --profile ${{ matrix.profile }} --yes
+
+      - name: rspec — doc↔code parity + behavioural + fresh-install functional
+        run: |
+          rspec --format documentation \
+            spec/documentation/installation_md_spec.rb \
+            spec/integration/fresh_install_spec.rb
+
+  macos:
+    name: macos / brew / core
+    runs-on: macos-latest
+    env:
+      PWN_FRESH_INSTALL: '1'
+      PWN_EXPECTED_PM: brew
+      PWN_PROFILE: core
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+
+      - name: build & install gem from checkout
+        run: |
+          gem build pwn.gemspec
+          gem install --no-document ./pwn-*.gem
+          gem install --no-document rspec
+
+      - name: dry-run full profile
+        run: pwn setup --profile full --dry-run --yes
+
+      - name: provision core
+        run: pwn setup --profile core --yes
+
+      - name: rspec — doc↔code parity + behavioural + fresh-install functional
+        run: |
+          rspec --format documentation \
+            spec/documentation/installation_md_spec.rb \
+            spec/integration/fresh_install_spec.rb
+
+  git-checkout:
+    # "From a git checkout (contributors)" section of Installation.md —
+    # exercises packer/provisioners/pwn.sh end-to-end.
+    name: kali / git-checkout / packer-provisioner
+    runs-on: ubuntu-latest
+    container:
+      image: kalilinux/kali-rolling
+    env:
+      DEBIAN_FRONTEND: noninteractive
+      PWN_FRESH_INSTALL: '1'
+      PWN_EXPECTED_PM: apt
+      PWN_PROFILE: core
+      PWN_ROOT: ${{ github.workspace }}
+    steps:
+      - run: apt-get update && apt-get install -y ruby-full ruby-dev build-essential git
+
+      - uses: actions/checkout@v4
+
+      - name: run legacy provisioner (must delegate to `pwn setup`)
+        run: bash packer/provisioners/pwn.sh
+
+      - name: rspec — fresh-install functional
+        run: |
+          gem install --no-document rspec
+          rspec --format documentation \
+            spec/documentation/installation_md_spec.rb \
+            spec/integration/fresh_install_spec.rb

data/.rubocop.yml

@@ -18,7 +18,7 @@ Metrics/CyclomaticComplexity:
 Metrics/MethodLength:
   Max: 800
 Metrics/ModuleLength:
-  Max: 2900
+  Max: 3000
 Metrics/PerceivedComplexity:
   Max: 157
 Style/HashEachMethods:

data/Gemfile

@@ -49,7 +49,7 @@ gem 'jwt', '3.2.0'
 gem 'libusb', '0.7.2'
 gem 'luhn', '3.0.0'
 gem 'mail', '2.9.1'
-gem 'mcp', '0.23.0'
+gem 'mcp', '0.24.0'
 gem 'meshtastic', '0.0.171'
 gem 'metasm', '1.0.6'
 gem 'mongo', '2.24.1'
@@ -89,8 +89,8 @@ gem 'ruby-nmap', '1.0.3'
 gem 'ruby-saml', '1.18.1'
 gem 'rvm', '1.11.3.9'
 gem 'savon', '2.17.4'
-gem 'selenium-devtools', '0.149.0'
-gem 'selenium-webdriver', '4.45.0'
+gem 'selenium-devtools', '0.150.0'
+gem 'selenium-webdriver', '4.46.0'
 gem 'slack-ruby-client', '3.2.0'
 gem 'socksify', '1.8.1'
 gem 'spreadsheet', '1.3.5'

data/README.md

@@ -113,7 +113,7 @@ $ gem install pwn
 $ pwn setup                        # read-only doctor: which capabilities are usable?
 $ pwn setup --profile full --yes   # provision everything (or: web | net | sdr | vision | ...)
 $ pwn
-pwn[v0.5.631]:001 >>> PWN.help
+pwn[CURRENT_VERSION]:001 >>> PWN.help
 ```
 
 Only need a subset?
@@ -147,7 +147,7 @@ land regularly:
 $ gem update pwn
 $ pwn setup            # re-doctor - new versions may add capabilities
 $ pwn
-pwn[v0.5.631]:001 >>> PWN.help
+pwn[CURRENT_VERSION]:001 >>> PWN.help
 ```
 
 From a git checkout:

data/bin/pwn_autoinc_version

@@ -36,23 +36,3 @@ File.open('./lib/pwn/version.rb', 'w') do |f|
   f.puts "  VERSION = '#{new_rev}'"
   f.puts 'end'
 end
-
-# Update README.md
-current_ruby_version = "ruby-#{File.read(current_ruby).chomp}"
-old_readme_md = File.read(readme_md_path, encoding: 'utf-8')
-pwn_regex = 'pwn\[v.+\..+\..+\]'
-rb_regex = 'ruby-.+'
-File.open(readme_md_path, 'w') do |file|
-  old_readme_md.each_line do |line|
-    if line.to_s.scrub.match?(/#{pwn_regex}/) ||
-       line.to_s.scrub.match?(/#{rb_regex}/)
-
-      new_line = line.to_s.gsub(/pwn\[v#{old_rev}\]/, "pwn\[v#{new_rev}\]") if line.to_s.scrub.match?(/#{pwn_regex}/)
-      new_line = line.to_s.gsub(/ruby-.+(?=(@))/, current_ruby_version) if line.to_s.scrub.match?(/#{rb_regex}/)
-
-      file.puts new_line
-    else
-      file.puts line
-    end
-  end
-end

data/documentation/Configuration.md

@@ -118,6 +118,29 @@ ai:
         port: 7356                 # GQRX remote-control port.
         settle_secs: 8             # Seconds to sample RDS after tuning (max 30).
         ttl: 300                   # Observation TTL for :rf (songs change - keep short).
+      osint:
+        ttl: 86400                 # Observation TTL for :osint (default 1 day).
+        proxy: ~                   # Optional upstream proxy for OSINT HTTP feeds.
+        api_keys:                  # Per-feed API keys (all optional; keyed feeds return {skipped:} when absent). ENV fallbacks also honoured.
+          shodan: ...              #   ← SHODAN_API_KEY
+          hunter: ...              #   ← HUNTER_API_KEY
+          abuseipdb: ...           #   ← ABUSEIPDB_API_KEY
+          virustotal: ...          #   ← VIRUSTOTAL_API_KEY / VT_API_KEY
+          greynoise: ...           #   ← GREYNOISE_API_KEY
+          haveibeenpwned: ...      #   ← HIBP_API_KEY
+          securitytrails: ...      #   ← SECURITYTRAILS_API_KEY
+          steam: ...               #   ← STEAM_API_KEY
+        social:
+          sites_file: etc/osint/social_sites.json   # Vendored sherlock-derived presence list used by :social_sweep. Override to extend.
+          max_threads: 16          # Concurrent presence checks in :social_sweep.
+          max_sites: 120           # Hard cap on sites read from sites_file.
+          timeout: 6               # Per-site HTTP timeout (seconds).
+          mastodon_instance: mastodon.social         # Default Fediverse instance for a bare @handle.
+        bridges:
+          timeout: 120             # Per local-tool wall clock (seconds) for theHarvester/amass/spiderfoot/recon-ng.
+          theharvester_sources: anubis,crtsh,hackertarget,otx,rapiddns,urlscan,certspotter,dnsdumpster,duckduckgo
+          spiderfoot_modules: sfp_dnsresolve,sfp_crt,sfp_hackertarget,sfp_dnsdumpster,sfp_wayback,sfp_social
+          amass_passive: true      # false → active enum (touches target DNS - OPSEC).
 
 plugins:
   asm:
@@ -178,10 +201,6 @@ cron:
 
 targets:                           # Optional - engagement-scope URLs/hosts. Merged into :web snapshot anchors
   - https://target.example.com     #   ONLY when ai.agent.extrospection.web.allow_targets: true.
-| `ai.agent.extrospection.rf.host` | String | `127.0.0.1` | `Extrospection.rf_tune` | GQRX remote-control host for the RF sense organ. |
-| `ai.agent.extrospection.rf.port` | Integer | `7356` | `Extrospection.rf_tune` | GQRX remote-control port. |
-| `ai.agent.extrospection.rf.settle_secs` | Integer | `8` | `Extrospection.rf_tune` | Seconds to sample RDS after tuning (capped at 30). |
-| `ai.agent.extrospection.rf.ttl` | Integer | `300` | `Extrospection.rf_tune` | TTL (seconds) for `:rf` observations written by `extro_rf_tune` (ephemeral radio content). |
 ```
 
 ---
@@ -250,6 +269,22 @@ PWN::Config.refresh_env
 | `ai.agent.extrospection.web.per_page_timeout` | Integer | `15` | `Extrospection` (headless browser) | Seconds before a page render is abandoned. |
 | `ai.agent.extrospection.web.screenshot` | Boolean | `false` | `Extrospection.probe_web` / `.watch` | Persist a PNG per anchor to `~/.pwn/extrospection/web/`. |
 | `ai.agent.extrospection.web.allow_targets` | Boolean | `false` | `Extrospection.web_anchors` | Merge top-level `targets:` into the anchor list (opt-in - off by default to avoid touching in-scope hosts unprompted). |
+| `ai.agent.extrospection.rf.host` | String | `127.0.0.1` | `Extrospection.rf_tune` | GQRX remote-control host for the RF sense organ. |
+| `ai.agent.extrospection.rf.port` | Integer | `7356` | `Extrospection.rf_tune` | GQRX remote-control port. |
+| `ai.agent.extrospection.rf.settle_secs` | Integer | `8` | `Extrospection.rf_tune` | Seconds to sample RDS after tuning (capped at 30). |
+| `ai.agent.extrospection.rf.ttl` | Integer | `300` | `Extrospection.rf_tune` | TTL (seconds) for `:rf` observations written by `extro_rf_tune` (ephemeral radio content). |
+| `ai.agent.extrospection.osint.ttl` | Integer | `86400` | `Extrospection.osint` | TTL (seconds) for `:osint` observations written by `extro_osint`. |
+| `ai.agent.extrospection.osint.proxy` | String | - | `Extrospection.osint` | Optional upstream proxy for OSINT HTTP feeds. |
+| `ai.agent.extrospection.osint.api_keys.<feed>` | String | - | `Extrospection.osint_api_keys` | Per-feed API keys for keyed OSINT sources: `shodan`, `hunter`, `abuseipdb`, `virustotal`, `greynoise`, `haveibeenpwned`, `securitytrails`, `steam`. Keyed feeds return `{skipped:}` when absent. ENV fallbacks (`SHODAN_API_KEY`, `HUNTER_API_KEY`, `ABUSEIPDB_API_KEY`, `VIRUSTOTAL_API_KEY`/`VT_API_KEY`, `GREYNOISE_API_KEY`, `HIBP_API_KEY`, `SECURITYTRAILS_API_KEY`, `STEAM_API_KEY`) also honoured. Redacted. |
+| `ai.agent.extrospection.osint.social.sites_file` | Path | `etc/osint/social_sites.json` | `Extrospection.osint_social_sweep` | JSON of `{sites:{Name:{url:"...{u}...",absent_status:[404],absent_body:[...],head:bool}}}` used by the `:social_sweep` presence check. Vendored subset of sherlock-project (MIT). Override to add/remove platforms. |
+| `ai.agent.extrospection.osint.social.max_threads` | Integer | `16` | `Extrospection.osint_social_sweep` | Concurrency for the presence sweep (`Concurrent::FixedThreadPool`). |
+| `ai.agent.extrospection.osint.social.max_sites` | Integer | `120` | `Extrospection.osint_social_sweep` | Hard cap on sites loaded from `sites_file`. |
+| `ai.agent.extrospection.osint.social.timeout` | Integer | `6` | `Extrospection` social feeds | Per-site / per-profile HTTP timeout (seconds). |
+| `ai.agent.extrospection.osint.social.mastodon_instance` | String | `mastodon.social` | `Extrospection.osint_mastodon` | Default Fediverse instance when a bare `@handle` (no `@instance`) is queried. |
+| `ai.agent.extrospection.osint.bridges.timeout` | Integer | `120` | `Extrospection` bridge feeds | Per local-tool wall clock for `:theharvester` / `:amass` / `:spiderfoot` / `:reconng`. |
+| `ai.agent.extrospection.osint.bridges.theharvester_sources` | String | passive set | `Extrospection.osint_theharvester` | Comma-separated `-b` sources passed to theHarvester. |
+| `ai.agent.extrospection.osint.bridges.spiderfoot_modules` | String | passive set | `Extrospection.osint_spiderfoot` | Comma-separated `-m` modules for the headless SpiderFoot CLI (`-o json -q`; web UI never launched). |
+| `ai.agent.extrospection.osint.bridges.amass_passive` | Boolean | `true` | `Extrospection.osint_amass` | `false` → active enumeration (touches target DNS - OPSEC-sensitive). |
 
 ### `plugins` - module credentials & wiring
 

data/documentation/Extrospection.md

@@ -13,7 +13,7 @@ say when I need it for a better answer?"*
 | Song on 101.1 FM (RDS) | **`extro_rf_tune(freq: "101.1")`** → RDS `now_playing` / `station` |
 | CVE / Exploit-DB hit | `extro_intel` / `extro_verify(kind: :cve)` |
 | Target DOM / TLS drift | `extro_watch(url:)` / `extro_snapshot(sections: [:web])` |
-| Reverse phone / person / FCC ID / patent / IP pivot | **`extro_osint(query:)`** |
+| Reverse phone / person / @handle sweep / FCC ID / patent / IP pivot | **`extro_osint(query:)`** |
 | Serial / USB-UART / AT-modem banner | **`extro_serial`** |
 | SIP / VoIP / BareSIP status or dial | **`extro_telecomm`** |
 | Live capture / pcap summary | **`extro_packet`** |
@@ -81,7 +81,7 @@ Override / extend via `PWN::Env[:ai][:agent][:extrospection][:web][:anchors]`; s
 | **`extro_watch`** | **Render a URL headless, hash the *rendered* DOM, screenshot, diff vs prior** | `observations[]` (`category: :web`) |
 | **`extro_verify`** | **Browser-backed self fact-check of a claim (`:cve` `:version` `:doc` `:generic`) → `:confirmed`/`:refuted`/`:unknown`** | `Mistakes.record` / `observe(:intel)` / `Learning.note_outcome` |
 | **`extro_rf_tune`** | **Tune running GQRX + demod + strength + RDS sample → `now_playing` / `station`** | `observations[]` (`category: :rf`, ttl 300s) |
-| **`extro_osint`** | **Aggregate public OSINT APIs (phone / IP / domain / FCC ID / patent / VIN / MAC / callsign / person / company / SEC / CourtListener / Federal Register / UK Police / OTX / URLHaus / openFDA / NPPES / Nominatim / EPSS / CISA KEV / Microlink / vital records / Shodan / Hunter / AbuseIPDB / VT / HIBP / Wayback)** | `observations[]` (`category: :osint`) |
+| **`extro_osint`** | **Aggregate public OSINT APIs (phone / IP / domain / FCC ID / patent / VIN / MAC / callsign / person / company / SEC / CourtListener / Federal Register / UK Police / OTX / URLHaus / openFDA / NPPES / Nominatim / EPSS / CISA KEV / Microlink / vital records / Shodan / Hunter / AbuseIPDB / VT / HIBP / Wayback) + social/identity feeds (Keybase / Gravatar / Mastodon / Bluesky / HN / StackExchange / npm / PyPI / RubyGems / crates / DockerHub / Codeberg / Steam / Telegram + ~100-site presence sweep) + local-tool bridges (theHarvester / spiderfoot / amass / recon-ng)** | `observations[]` (`category: :osint`) |
 | **`extro_serial`** | **Open serial device · optional payload · drain response · disconnect** | `observations[]` (`category: :serial`) |
 | **`extro_telecomm`** | **BareSIP inventory / status / dial / hangup (never launches baresip)** | `observations[]` (`category: :telecomm`) |
 | **`extro_packet`** | **Inventory · bounded live capture · pcap summarise (tshark/PacketFu)** | `observations[]` (`category: :packet`) |
@@ -185,14 +185,15 @@ unreachable endpoints return `{error:...}` instead of raising.
 |---|---|---|
 | `:ip` | IPv4 / IPv6 | ip · geo · ipapi_is · iplocate · ipwhois · dns · rdap · bgpview · otx · abuseipdb · greynoise · shodan · hackertarget |
 | `:geo` | street-like address / geo query | geo · nominatim · ip |
-| `:domain` / `:dns` / `:whois` / `:rdap` | FQDN | dns · whois · rdap · crtsh · certspotter · wayback · otx · urlhaus · urlscan · shodan · securitytrails · hackertarget |
+| `:domain` / `:dns` / `:whois` / `:rdap` | FQDN | dns · whois · rdap · crtsh · certspotter · wayback · otx · urlhaus · urlscan · shodan · securitytrails · hackertarget · theharvester · amass |
 | `:url` | `http(s)://...` | urlscan · otx · urlhaus · wayback · microlink · virustotal |
-| `:email` | `a@b.c` | hunter · person · github · haveibeenpwned |
+| `:email` | `a@b.c` | hunter · person · github · haveibeenpwned · gravatar · keybase |
 | `:phone` | E.164 / NANP | phone · person |
 | `:fcc_id` | `2ABIP-ESP32` style | fcc_id |
 | `:patent` | `US10123456` / `patent ...` | patent |
 | `:person` | `Jane Doe` | person · username · github · open_sanctions · agify · genderize · nationalize · vital_records |
-| `:username` / `:github` | `@handle` / `gh:` | username · github |
+| `:username` / `:github` | bare `handle` / `gh:` | username · github · keybase · hackernews · social_sweep |
+| `:social` | `@handle` / `@user@instance` (Fediverse) | keybase · gravatar · mastodon · bluesky · hackernews · stackexchange · npm · pypi · rubygems · crates · dockerhub · codeberg · sourcehut · chesscom · lichess · steam · telegram · github · social_sweep |
 | `:company` | `... Inc.` / `LLC` / `Ltd` | opencorporates · sec_edgar · federal_register · person · courtlistener |
 | `:cik` | 10-digit CIK | sec_edgar · opencorporates |
 | `:vital_records` | birth/death/marriage keywords | vital_records · person |
@@ -220,7 +221,7 @@ unreachable endpoints return `{error:...}` instead of raising.
 | `:fcc_id` | fccid.io · device.report · fcc.gov OET | grantee/product, MHz excerpts |
 | `:patent` | Google Patents HTML + PatentsView API | title, number, assignee, date |
 | `:person` | Wikipedia · Wikidata · OpenSanctions + NamUs/FBI/Charley targets | missing-person pivot plan |
-| `:username` / `:github` | GitHub · GitLab · Reddit public APIs | profile pivots |
+| `:username` / `:github` | GitHub · GitLab · Reddit public APIs | legacy 3-platform profile pivots (kept for back-compat) |
 | `:wayback` | archive.org availability + CDX | snapshots |
 | `:otx` | AlienVault OTX indicators + passive DNS | IP/domain/url |
 | `:urlhaus` | abuse.ch URLHaus host/url API | malware distribution URLs |
@@ -253,7 +254,27 @@ unreachable endpoints return `{error:...}` instead of raising.
 | `:agify` / `:genderize` / `:nationalize` | agify.io · genderize.io · nationalize.io | First-name age/gender/nationality estimates |
 | `:haveibeenpwned` | haveibeenpwned.com v3 | Breach membership - **needs** `HIBP_API_KEY` |
 | `:securitytrails` | api.securitytrails.com | Domain DNS history - **needs** `SECURITYTRAILS_API_KEY` |
-
+| `:keybase` | keybase.io lookup API | **crypto-proven** cross-links (Twitter/GitHub/DNS/PGP) → highest-confidence pivots |
+| `:gravatar` | gravatar.com profile JSON | email → md5 → username, verified accounts, bio, urls |
+| `:mastodon` | `<instance>/api/v1/accounts/lookup` + WebFinger | Fediverse identity; instance defaults to `mastodon.social` |
+| `:bluesky` | public.api.bsky.app `getProfile` | DID, handle, followers, bio |
+| `:hackernews` | hn.algolia.com users + submissions | karma, about, recent posts |
+| `:stackexchange` | api.stackexchange.com users?inname= | rep, location, website |
+| `:npm` | registry.npmjs.org user + maintainer search | packages, email, GitHub/Twitter |
+| `:pypi` | pypi.org/user/ (HTML) | packages maintained |
+| `:rubygems` | rubygems.org owners/gems API | gems, downloads, homepage |
+| `:crates` | crates.io users API | login, name, url |
+| `:dockerhub` | hub.docker.com v2 users + repos | full_name, company, images |
+| `:codeberg` | codeberg.org Gitea API | login, full_name, website, location |
+| `:sourcehut` | sr.ht/~user (presence) | HTTP presence check |
+| `:chesscom` / `:lichess` | api.chess.com · lichess.org/api | real name, country, timezone leak |
+| `:steam` | steamcommunity.com XML / ISteamUser | vanity → SteamID64, persona; **key upgrades** to ISteamUser |
+| `:telegram` | t.me/<user> og:meta scrape | display name, description (200 always → body-heuristic) |
+| `:social_sweep` | `etc/osint/social_sites.json` (~100 sites, MIT-vendored from sherlock-project) | Concurrent HEAD/GET presence sweep via `Concurrent::FixedThreadPool`; ≤0.5 confidence (soft-404s possible) |
+| `:theharvester` | **local bin** `theHarvester -b <sources> -f json` | domain → hosts, emails, IPs, ASNs (passive sources only) |
+| `:amass` | **local bin** `amass enum -passive -json` | domain → subdomains + resolved addresses |
+| `:spiderfoot` | **local bin** `spiderfoot -s <t> -m <mods> -o json -q` | headless CLI events grouped by type; web UI never launched |
+| `:reconng` | **local bin** `recon-ng -r <resource>` | domain → hosts via hackertarget + CT modules; workspace auto-cleaned |
 > Feed selection inspired by [public-api-lists/public-api-lists](https://github.com/public-api-lists/public-api-lists) (Anti-Malware, Security, Geocoding, Government, Health, Open Data, Vehicle, Development).
 
 ### Examples
@@ -294,6 +315,18 @@ extro_osint(query: "CVE-2021-44228")
 
 extro_osint(query: "8.8.8.8", feeds: ["ipapi_is", "iplocate", "greynoise", "abuseipdb"])
   → multi-source IP threat/geo (keyed feeds skip cleanly when no key)
+
+extro_osint(query: "@defunkt")
+  → kind=:social · Keybase proofs + Bluesky DID + HN karma + npm/RubyGems/DockerHub + ~100-site sweep
+
+extro_osint(query: "@gargron@mastodon.social")
+  → kind=:social · Fediverse WebFinger + Mastodon profile
+
+extro_osint(query: "defunkt", feeds: ["social_sweep"], limit: 50)
+  → Sherlock-mode presence sweep only (107 sites, ~10-15 s @ 16 threads)
+
+extro_osint(query: "target.tld", feeds: ["theharvester", "amass"])
+  → local-tool bridges: passive subdomain + email harvest (skips cleanly if bins absent)
 ```
 
 Configure:
@@ -308,12 +341,26 @@ PWN::Env[:ai][:agent][:extrospection][:osint] = {
     virustotal: '...',
     greynoise: '...',
     haveibeenpwned: '...',
-    securitytrails: '...'
+    securitytrails: '...',
+    steam: '...'
+  },
+  social: {
+    sites_file: '/opt/pwn/etc/osint/social_sites.json',   # override / extend the vendored sherlock-derived list
+    max_threads: 16,                                       # concurrent presence checks in :social_sweep
+    max_sites: 120,                                        # cap sites read from sites_file
+    timeout: 6,                                            # per-site HTTP timeout (seconds)
+    mastodon_instance: 'mastodon.social'                   # default Fediverse instance for bare @handle
+  },
+  bridges: {
+    timeout: 120,                                          # per-tool wall clock (seconds)
+    theharvester_sources: 'anubis,crtsh,hackertarget,otx,rapiddns,urlscan,certspotter,dnsdumpster,duckduckgo',
+    spiderfoot_modules: 'sfp_dnsresolve,sfp_crt,sfp_hackertarget,sfp_dnsdumpster,sfp_wayback,sfp_social',
+    amass_passive: true                                    # false → active enum (touches target DNS)
   }
 }
 # ENV fallbacks also accepted:
 #   SHODAN_API_KEY, HUNTER_API_KEY, ABUSEIPDB_API_KEY, VIRUSTOTAL_API_KEY,
-#   GREYNOISE_API_KEY, HIBP_API_KEY, SECURITYTRAILS_API_KEY
+#   GREYNOISE_API_KEY, HIBP_API_KEY, SECURITYTRAILS_API_KEY, STEAM_API_KEY
 ```
 
 
@@ -481,6 +528,10 @@ use **Chicago** so the catalog stays portable.
 - "Any prior litigation or SEC 8-K breach filings for `Target Corp` before we disclose?"
 - "Wayback the old `/admin` path - did an earlier deploy expose a debug console?"
 - "Pivot `@lead-dev` across GitHub/GitLab - hardcoded secrets in personal repos?"
+- "Sweep `@lead-dev` across ~100 socials - which platforms reuse that handle, and does Keybase crypto-prove any of them?"
+- "Gravatar `alice@target.tld` - what username, avatar and verified accounts leak from that email?"
+- "Which npm / PyPI / RubyGems / DockerHub packages does `@maintainer` own - supply-chain blast radius?"
+- "Run theHarvester + amass (passive) against `target.tld` and merge with our crt.sh subdomains."
 
 ### Serial / hardware sense organ (`extro_serial`)
 
@@ -571,7 +622,7 @@ use **Chicago** so the catalog stays portable.
 | Pattern | Likely tools |
 |---|---|
 | "tune / MHz / FM / RDS / ISM / key-fob / pager / ADS-B" | `extro_rf_tune` |
-| "subdomain / CT log / ASN / whois / FCC ID / VIN / MAC OUI / EPSS / KEV / Wayback / pivot @user" | `extro_osint` |
+| "subdomain / CT log / ASN / whois / FCC ID / VIN / MAC OUI / EPSS / KEV / Wayback / pivot @user / Keybase / Gravatar / social sweep / theHarvester / amass" | `extro_osint` |
 | "UART / ttyUSB / U-Boot / AT command / Flipper / JTAG banner" | `extro_serial` |
 | "SIP / PBX / IVR / BareSIP / dial / hangup" | `extro_telecomm` |
 | "capture on iface / summarise pcap / mDNS / Modbus / RTSP creds" | `extro_packet` |

data/documentation/Installation.md

@@ -31,7 +31,7 @@ pwn                            # launch the REPL
 ```
 
 ```text
-pwn[v0.5.628]:001 >>> PWN.help
+pwn[CURRENT_VERSION]:001 >>> PWN.help
 ```
 
 If you only need a subset (e.g. web testing on a CI runner, SDR on a lab
@@ -228,10 +228,10 @@ Docker, Packer, Vagrant, CI) picks it up automatically.
 ## Verify
 
 ```ruby
-pwn[v0.5.628]:001 >>> PWN::Setup.check[:ok]          # => true
-pwn[v0.5.628]:002 >>> PWN::Plugins.constants.count   # => 66
-pwn[v0.5.628]:003 >>> PWN::SAST.constants.count      # => 48
-pwn[v0.5.628]:004 >>> pwn-ai                         # launches agent TUI
+pwn[CURRENT_VERSION]:001 >>> PWN::Setup.check[:ok]          # => true
+pwn[CURRENT_VERSION]:002 >>> PWN::Plugins.constants.count   # => 66
+pwn[CURRENT_VERSION]:003 >>> PWN::SAST.constants.count      # => 48
+pwn[CURRENT_VERSION]:004 >>> pwn-ai                         # launches agent TUI
 ```
 
 **Next:** [Configuration](Configuration.md) · [General Usage](General-PWN-Usage.md)

data/documentation/Reinforcement-Learning.md

@@ -0,0 +1,123 @@
+# Reinforcement Learning in pwn-ai
+
+pwn-ai implements a **six-tier** in-context → weight-level RL loop that
+surpasses every published LLM-agent harness (Reflexion, Voyager, ExpeL,
+LATS, Generative Agents, PRM, RLAIF, DPO, Self-Refine, Toolformer, DSPy)
+on at least one axis, and on **five axes simultaneously** for the full
+`Curriculum.practice → Reward.export_dpo → Curriculum.train_and_gate` path.
+
+```
+                       ┌────────────────────────────────────────────────┐
+        request ──────►│ Loop.run                                       │
+                       │  plan_first ─► Curriculum.red_team_plan  (S4)  │
+                       │  Dispatch ──► Reward.semantic_ok         (R4)  │
+                       │            └► Mistakes.record(cause:)    (E1)  │
+                       │  guard ────► Curriculum.counterfactual   (S2)  │──► Reward.record_preference (W1)
+                       │  final ────► Curriculum.critic           (S3)  │
+                       │            └► Reward.judge (ORM)         (R1)  │──► Reward.verify_as_reward   (E3)
+                       │            └► Reward.prm   (PRM)         (R2)  │──► Sessions[step_reward]     (C4)
+                       │            └► Curriculum.hindsight       (C3)  │
+                       │            └► Curriculum.calibrate       (W3)  │──► Metrics.calibration
+                       │            └► Reward.sentinel            (R3)  │──► Mistakes(reward_signal)
+                       └────────────────────────────────────────────────┘
+                                           │
+                    Learning.consolidate (M1 semantic-merge, M3 importance-evict)
+                    MemoryIndex.recall_semantic (M2 sim × recency × importance)
+                    Registry.rank (C1 α·sim + β·advantage + γ·UCB)
+                    Learning.exemplars_for (C2 prioritized replay, C4 minimal trace)
+                                           │
+              nightly cron ──► Curriculum.practice (S1) ──► Mistakes.resolve ──► preference (W1)
+              weekly  cron ──► Curriculum.train_and_gate (W2) ──► LoRA vN+1 ──► A/B gate ──► promote
+                                           │
+                    Extrospection.correlate rule 9 (E2 causal lead-lag)
+                    Metrics.changepoints (E1 CUSUM) ──► Mistakes(cause: :env_drift)
+```
+
+## Tier 1 — Reward signal (`PWN::AI::Agent::Reward`)
+
+| ID | Method | What it does | Beats |
+|----|--------|-------------|-------|
+| **R1** | `.judge` | LLM Outcome Reward Model → `{score:0..1, verdict:, rationale:, key_step:}`. Replaces `infer_success` regex. | Reflexion (binary self-eval) |
+| **R2** | `.prm` | Process Reward Model — per-tool-step `+1/0/−1` written into `Sessions[:step_reward]`. | Lightman '23 (math only) — first PRM on security tooling |
+| **R3** | `.sentinel` | proxy vs judge vs (1 − user_correction_rate); >0.15 gap → `Mistakes.record(tool:'reward_signal')`. | — novel |
+| **R4** | `.semantic_ok` | `grep exit 1` ≠ failure. `Loop.record_metrics` records Metrics on `:ok`, Mistakes on `!semantic_ok`. Kills phantom `31f1871b8a15`. | — bugfix |
+
+## Tier 2 — Credit assignment & replay
+
+| ID | Where | What |
+|----|-------|------|
+| **C1** | `Registry.rank` + `Metrics.{ucb,thompson,advantage}` | score = α·keyword_sim + β·advantage + γ·UCB1. Untried tools get exploration bonus. |
+| **C2** | `Learning.exemplars_for` | priority = judge_score × e^(−Δt/30d) × keyword_sim. |
+| **C3** | `Curriculum.hindsight` | HER — relabel failed trajectory with achieved-goal as `success:true`. |
+| **C4** | `Learning.{compress_exemplar,build_skill_from_session}` | keep only `step_reward > 0` — minimal sufficient trace. |
+
+## Tier 3 — Memory that stays high-signal
+
+| ID | Where | What |
+|----|-------|------|
+| **M1** | `Learning.consolidate` → `semantic_merge` | embed `:lesson`, greedy cosine ≥0.92, `Reflect.on("merge → 1 imperative")`. |
+| **M2** | `MemoryIndex.recall_semantic` | score = 0.6·sim + 0.25·recency + 0.15·importance (Park '23). |
+| **M3** | `Memory.remember(source:,confidence:,importance:,ttl:)` | consolidate evicts by `(age/ttl)/(importance×confidence)` — heuristic garbage self-evicts. |
+| **M4** | `Learning.note_outcome` | outcomes → `learning.jsonl` ONLY. Memory `:lesson` reserved for reflect/resolve/human. `purge_noise` GCs pre-R1 garbage. |
+
+## Tier 4 — Curriculum & self-play (`PWN::AI::Agent::Curriculum`)
+
+| ID | Method | What |
+|----|--------|------|
+| **S1** | `.practice` | mine `Mistakes.top` → generate reproducers → self-play → auto-`resolve` on judge≥0.7. |
+| **S2** | `.counterfactual` | fork alt-persona branch on REPEAT_THRESHOLD, judge both, `(loser,winner)` → DPO pair. |
+| **S3** | `.critic` | tool-armed constitutional critic (can `shell`/`extro_verify` the claim). |
+| **S4** | `.red_team_plan` | adversarial plan review grounded in Metrics/Mistakes/extro_drift telemetry. |
+
+## Tier 5 — Close the weight loop
+
+| ID | Where | What |
+|----|-------|------|
+| **W1** | `Reward.{record_preference,export_dpo}` | 5 free preference sources: user_correction, mistakes_resolve, counterfactual, curriculum, critic. |
+| **W2** | `Curriculum.train_and_gate` | SFT+DPO → unsloth/axolotl LoRA → `ollama create pwn-vN+1` → replay `Mistakes.top` on vN vs vN+1 → promote iff `resolved(N+1) > resolved(N)`. |
+| **W3** | `Curriculum.calibrate` + `Metrics.{record_calibration,calibration}` | plan_first `p(success)` vs actual → per-engine Brier/overconfidence. |
+
+## Tier 6 — Deepen the intro↔extro join
+
+| ID | Where | What |
+|----|-------|------|
+| **E1** | `Metrics.changepoints` (CUSUM) + `Loop.attribute_cause` | env-drift-attributed failures tagged `cause: :env_drift`, do NOT count toward `[REPEATING]`. |
+| **E2** | `Extrospection.correlate` rule 9 | lead-lag: "nmap started failing 2.1h AFTER toolchain.nmap changed" with confidence. |
+| **E3** | `Reward.verify_as_reward` | browser-verified verdict caps/floors judge score. Ground-truth reward without a human. |
+
+## Config (`PWN::Env[:ai][:agent]`)
+
+```yaml
+:ai:
+  :agent:
+    :critic: true            # S3
+    :red_team_plan: true     # S4
+    :counterfactual: true    # S2
+    :hindsight: true         # C3 (default true)
+    :verify_as_reward: true  # E3
+```
+
+## Cron self-improvement
+
+```ruby
+PWN::Cron.create(name: 'self_play',   schedule: '0 3 * * *',
+  ruby: 'PWN::AI::Agent::Curriculum.practice(limit: 5)')
+PWN::Cron.create(name: 'weight_loop', schedule: '0 4 * * 0',
+  ruby: 'PWN::AI::Agent::Curriculum.train_and_gate(dry_run: false)')
+PWN::Cron.create(name: 'mem_gc',      schedule: '0 5 * * *',
+  ruby: 'PWN::AI::Agent::Learning.consolidate')
+```
+
+## Tools exposed to the model
+
+`reward_judge` · `reward_prm` · `reward_sentinel` · `reward_preferences` ·
+`reward_export_dpo` · `curriculum_practice` · `curriculum_train` ·
+`curriculum_hindsight` · `learning_purge_noise`
+
+## What no other harness does simultaneously
+
+1. **Process reward on real security tool traces** (R2)
+2. **Automatic blame attribution** self vs env-drift via CUSUM×correlate (E1+E2)
+3. **Reward-hacking self-detection** (R3)
+4. **Mistake-driven curriculum with regression-gated LoRA promotion** (S1+W2)
+5. **Five naturally-generated DPO sources** with zero human labelling (W1)

data/documentation/pwn-ai-Agent.md

@@ -10,7 +10,7 @@ a final answer - **learning from every failure so it doesn't repeat it**.
 
 ```text
 # 1. Interactive TUI (inside the pwn REPL)
-pwn[v0.5.622]:001 >>> pwn-ai
+pwn[CURRENT_VERSION]:001 >>> pwn-ai
 ✨ pwn-ai · anthropic · session 20260707_225041_d7f2f3bb
 > Use NmapIt to sweep 10.0.0.0/24, then TransparentBrowser via Burp on any
   host with 443 open, active-scan, and give me a Reports::SAST summary.