ronin-wordlists 0.1.0.rc1

data/data/completions/ronin-wordlists.yml

@@ -0,0 +1,7 @@
+---
+ronin-wordlists remove: &remove
+  - $(ronin-wordlists list)
+ronin-wordlists rm: *remove
+ronin-wordlists update: &update
+  - $(ronin-wordlists list)
+ronin-wordlists up: *update

data/data/wordlists.yml

@@ -0,0 +1,260 @@
+#
+# YAML Schema
+# ===========
+#
+#     wordlist-name:
+#       :url: https://...
+#       :summary: ...
+#       :categories:
+#         - ...
+#         - ...
+#
+# Note: the wordlist's URL may be a regular `http://` or `https://` URL to a
+# file, or a `https://` Git URL (ex: `https://github.com/foo/bar.git`).
+#
+---
+alexa-top-1000:
+  :url: https://github.com/urbanadventurer/WhatWeb/blob/master/plugin-development/alexa-top-1000.txt
+  :summary: The Alexa Top 1000 domain names.
+  :categories:
+    - dns
+    - domains
+
+subdomains:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains.txt
+  :summary: List of common subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-100:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains-100.txt
+  :summary: Top 100 most common subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-500:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains-500.txt
+  :summary: Top 500 most common subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-1000:
+  :url: https:/raw.githubusercontent./com/rbsec/dnscan/master/subdomains-1000.txt
+  :summary: Top 1000 most common subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-10000:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains-10000.txt
+  :summary: Top 10000 most common subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-uk-500:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains-uk-500.txt
+  :summary: Top 500 most common UK subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+subdomains-uk-1000:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/subdomains-uk-1000.txt
+  :summary: Top 1000 most common UK subdomain names used by the dnscan util.
+  :categories:
+    - dns
+    - subdomains
+
+suffixes:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/suffixes.txt
+  :summary: List of common domain suffixes used by the dnscan util.
+  :categories:
+    - dns
+    - public-suffixes
+
+tlds:
+  :url: https://raw.githubusercontent.com/rbsec/dnscan/master/tlds.txt
+  :summary: List of common TLDs used by the dnscan util.
+  :categories:
+    - dns
+    - tlds
+
+argon:
+  :url: http://www.mediafire.com/file/zd3zhzfezl9wd96/Argon_Wordlist_v2.gz
+  :summary: Really big password list.
+  :categories:
+    - passwords
+
+hashes.org-2012-2015:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2012-2015.7z
+  :summary: Hashes.org archive of passwords found in 2012 - 2015.
+  :categories:
+    - passwords
+
+hashes.org-2016:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2016.7z
+  :summary: Hashes.org archive of passwords found in 2016.
+  :categories:
+    - passwords
+
+hashes.org-2017:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2017.7z
+  :summary: Hashes.org archive of passwords found in 2017.
+  :categories:
+    - passwords
+
+hashes.org-2018:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2018.7z
+  :summary: Hashes.org archive of passwords found in 2018.
+  :categories:
+    - passwords
+
+hashes.org-2019:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2019.7z
+  :summary: Hashes.org archive of passwords found in 2019.
+  :categories:
+    - passwords
+
+hashes.org-2020:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2020.7z
+  :summary: Hashes.org archive of passwords found in 2020.
+  :categories:
+    - passwords
+
+hashes.org-2021:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashes.org-2021.txt
+  :summary: Hashes.org archive of passwords found in 2021.
+  :categories:
+    - passwords
+
+hashkiller-dict:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/hashkiller-dict.txt
+  :summary: Hashkiller dictionary from hashkiller.io.
+  :categories:
+    - passwords
+
+DailyQuiz:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/DailyQuiz.txt
+  :summary: Passwords from the DailyQuiz.me data breach.
+  :categories:
+    - passwords
+
+LiveJournal:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/LiveJournal.txt
+  :summary: Passwords from the LiveJournal.com data breach.
+  :categories:
+    - passwords
+
+Top120Million-probable-WPA:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/Top120Million-probable-WPA.txt
+  :summary: Old set of popular WPA passwords.
+  :categories:
+    - passwords
+    - wifi
+
+zxcvbn:
+  :url: https://hashmob.net/api/v2/downloads/research/wordlists/zxcvbn.txt
+  :summary: Realistic weak passwords.
+  :categories:
+    - passwords
+
+rockyou:
+  :url: https://github.com/brannondorsey/naive-hashcat/releases/download/data/rockyou.txt
+  :summary: Common passwords list.
+  :categories:
+    - passwords
+
+weakpass_3:
+  :url: https://download.weakpass.com/wordlists/1947/weakpass_3.7z
+  :summary: Updated Weakpass wordlist for general purpose.
+  :categories:
+    - passwords
+
+weakpass_3w:
+  :url: https://download.weakpass.com/wordlists/1950/weakpass_3w.7z
+  :summary:
+    Same as weakpass_3p, but passwords length limited from 8 to 24 chars.
+    Specially for fast WI-FI cracking.
+  :categories:
+    - passwords
+    - wifi
+
+weakpass_3p:
+  :url: https://download.weakpass.com/wordlists/1949/weakpass_3p.7z
+  :summary: Same as weakpass_3, but passwords contains only printalbe charsets.
+  :categories:
+    - passwords
+
+weakpass_3a:
+  :url: https://download.weakpass.com/wordlists/1948/weakpass_3a.7z
+  :summary:
+    The most complete compilation of wordlist's - more than 1500 in one.
+    Contains near 8 billion of passwords with length from 4 to 25.	
+  :categories:
+    - passwords
+
+directory-list-2-3-medium:
+  :url: https://github.com/daviddias/node-dirbuster/raw/master/lists/directory-list-2.3-medium.txt
+  :summary: Common directory name wordlist used for dirbusting.
+  :categories:
+    - dirs
+
+ws-dirs:
+  :url: https://raw.githubusercontent.com/3ndG4me/KaliLists/master/wfuzz/webservices/ws-dirs.txt
+  :summary: wfuzz webservices directories' wordlist.
+  :categories:
+    - dirs
+
+ws-files:
+  :url: https://raw.githubusercontent.com/3ndG4me/KaliLists/master/wfuzz/webservices/ws-files.txt
+  :summary: wfuzz webservices files' wordlist.
+  :categories:
+    - files
+
+xajkep-wordlists:
+  :url: https://github.com/xajkep/wordlists.git
+  :summary: xajkep's wordlists.
+  :categories:
+    - collection
+
+SecLists:
+  :url: https://github.com/danielmiessler/SecLists.git
+  :summary: The SecLists repository of pentester wordlists.
+  :categories:
+    - collection
+
+pry0cc:
+  :url: https://github.com/pry0cc/relevant-wordlist.git
+  :summary: Dynamically generated wordlists based off of news headlines.
+  :categories:
+    - collection
+
+fuzzdb:
+  :url: https://github.com/fuzzdb-project/fuzzdb.git
+  :summary: FuzzDB repository of fuzzing wordlists.
+  :categories:
+    - collection
+    - fuzzing
+
+kkrypt0nn:
+  :url: https://github.com/kkrypt0nn/wordlists.git
+  :summary: kkrypt0nn's collection of wordlists.
+  :categories:
+    - collection
+
+trickest:
+  :url: https://github.com/trickest/wordlists.git
+  :summary: trickest's collection of wordlists.
+  :categories:
+    - collection
+
+graphql-wordlist:
+  :url: https://github.com/Escape-Technologies/graphql-wordlist.git
+  :summary: GraphQL wordlists for Operations, field names, type names, etc.
+  :categories:
+    - collection
+    - graphql

data/gemspec.yml

@@ -0,0 +1,40 @@
+name: ronin-wordlists
+summary: A library and tool for managing wordlists.
+description: |
+  ronin-wordlists is a library and tool for managing wordlists. ronin-wordlists
+  can install and update wordlists, and contains a curated list of popular
+  wordlists and their download URLs.
+
+license: LGPL-3.0
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://ronin-rb.dev/
+has_yard: true
+
+metadata:
+  documentation_uri: https://ronin-rb.dev/docs/ronin-wordlists
+  source_code_uri:   https://github.com/ronin-rb/ronin-wordlists
+  bug_tracker_uri:   https://github.com/ronin-rb/ronin-wordlists/issues
+  changelog_uri:     https://github.com/ronin-rb/ronin-wordlists/blob/main/ChangeLog.md
+  rubygems_mfa_required: 'true'
+
+generated_files:
+ - data/completions/ronin-wordlists
+ - man/ronin-wordlists.1
+ - man/ronin-wordlists-search.1
+ - man/ronin-wordlists-completion.1
+ - man/ronin-wordlists-download.1
+ - man/ronin-wordlists-list.1
+ - man/ronin-wordlists-purge.1
+ - man/ronin-wordlists-remove.1
+ - man/ronin-wordlists-update.1
+
+required_ruby_version: ">= 3.0.0"
+
+dependencies:
+  wordlist: ~> 1.1
+  # Ronin dependencies:
+  ronin-core: ~> 0.2.0.rc1
+
+development_dependencies:
+  bundler: ~> 2.0

data/lib/ronin/wordlists/cache_dir.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+#
+# ronin-wordlists - A library and tool for managing wordlists.
+#
+# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
+#
+# ronin-wordlists is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-wordlists is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-wordlists.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/wordlists/wordlist_dir'
+require 'ronin/wordlists/exceptions'
+require 'ronin/core/home'
+
+require 'uri'
+require 'yaml'
+require 'yaml/store'
+require 'fileutils'
+
+module Ronin
+  module Wordlists
+    #
+    # Represents the cache directory.
+    #
+    # @api private
+    #
+    class CacheDir
+
+      include Enumerable
+
+      # The `~/.cache/ronin-wordlists/` directory where all repos are stored.
+      PATH = Core::Home.cache_dir('ronin-wordlists')
+
+      # The path to the cache directory.
+      #
+      # @return [String]
+      attr_reader :path
+
+      # The directory containing the downloaded wordlists.
+      #
+      # @return [WrodlistDir]
+      attr_reader :wordlist_dir
+
+      #
+      # Initializes the repository cache.
+      #
+      # @param [String] path
+      #   The path to the repository cache directory.
+      #
+      def initialize(path=PATH)
+        @path = path
+
+        @manifest_file = File.join(@path,'manifest.yml')
+        @wordlist_dir  = WordlistDir.new(File.join(@path,'wordlists'))
+
+        @manifest = load_manifest
+      end
+
+      # Mapping of wordlist `type:` values and their classes.
+      WORDLIST_TYPES = {
+        git:  WordlistRepo,
+        file: WordlistFile
+      }
+
+      #
+      # Accesses a wordlist file or repository from the cache directory.
+      #
+      # @param [String] name
+      #   The name of the wordlist file or repository.
+      #
+      # @return [WordlistRepo, WordlistFile]
+      #   The wordlist file or repository.
+      #
+      # @raise [WordlistNotFound]
+      #   No wordlist with the given name exists in the cache directory.
+      #
+      # @raise [InvalidManifestFile]
+      #   The `~/.cache/ronin-wordlists/manifest.yml` file contained invalid
+      #   YAML data.
+      #
+      def [](name)
+        unless (metadata = @manifest[name])
+          raise(WordlistNotFound,"wordlist not downloaded: #{name.inspect}")
+        end
+
+        type = metadata.fetch(:type) do
+          raise(InvalidManifestFile,"entry #{name.inspect} is missing a :type attribute")
+        end
+
+        url  = metadata.fetch(:url) do
+          raise(InvalidManifestFile,"entry #{name.inspect} is missing a :url attribute")
+        end
+
+        filename = metadata.fetch(:filename) do
+          raise(InvalidManifestFile,"entry #{name.inspect} is missing a :filename attribute")
+        end
+
+        path = File.join(@wordlist_dir.path,filename)
+
+        wordlist_class = WORDLIST_TYPES.fetch(type) do
+          raise(InvalidManifestFile,"unsupported wordlist type: #{type.inspect}")
+        end
+
+        return wordlist_class.new(path, url: url)
+      end
+
+      #
+      # Enumerates over every wordlist in the cache directory.
+      #
+      # @yield [name, wordlist]
+      #
+      # @yieldparam [WordlistFile, WordlistRepo] wordlist
+      #
+      # @return [Enumerator]
+      #   If no block is given an enumerator will be returned.
+      #
+      def each
+        return enum_for unless block_given?
+
+        @manifest.each_key do |name|
+          yield self[name]
+        end
+      end
+
+      #
+      # Lists the wordlists in the cache directory.
+      #
+      # @param [String] name
+      #   Optional file name to search for.
+      #
+      # @return [Array<String>]
+      #   The wordlist files within the wordlist directory.
+      #
+      def list(name='*')
+        @wordlist_dir.list(name)
+      end
+
+      #
+      # Opens a wordlist from the wordlist directory.
+      #
+      # @param [String] name
+      #   The wordlist file name.
+      #
+      # @return [Wordlist::File]
+      #   The opened wordlist file.
+      #
+      # @raise [WordlistNotFound]
+      #   No wordlist with the given name.
+      #
+      def open(name)
+        @wordlist_dir.open(name)
+      end
+
+      #
+      # Downloads a wordlist into the cache directory.
+      #
+      # @param [String, URI::HTTP] url
+      #   The wordlist URL.
+      #
+      # @return [WordlistFile, WordlistRepo]
+      #   The newly downloaded wordlist or the previously downloaded wordlist.
+      #
+      def download(url)
+        wordlist = @wordlist_dir.download(url)
+
+        update_manifest do |manifest|
+          manifest[wordlist.name] = {
+            type:     wordlist.type,
+            filename: wordlist.filename,
+            url:      wordlist.url.to_s
+          }
+        end
+
+        return wordlist
+      end
+
+      #
+      # Updates the wordlists in the cache directory.
+      #
+      def update
+        each(&:update)
+      end
+
+      #
+      # Deletes a wordlist from the cache directory.
+      #
+      # @param [String] name
+      #   The wordlist file or directory name to delete.
+      #
+      def remove(name)
+        wordlist = self[name]
+        wordlist.delete
+
+        update_manifest do |manifest|
+          manifest.delete(name)
+        end
+      end
+
+      #
+      # Purge all wordlists from the cache directory.
+      #
+      def purge
+        FileUtils.rm_rf(@path)
+      end
+
+      private
+
+      #
+      # Loads the manifest file.
+      #
+      # @return [Hash{String => Hash{Symbol => String}}]
+      #
+      def load_manifest
+        if File.file?(@manifest_file)
+          YAML.load_file(@manifest_file)
+        else
+          {}
+        end
+      end
+
+      #
+      # Updates both the {#manifest} and the `manifest.yml` file.
+      #
+      # @yield [manifest]
+      #
+      # @yieldparam [Hash] manifest
+      #
+      def update_manifest(&block)
+        yield @manifest
+        update_manifest_file(&block)
+      end
+
+      #
+      # Updates the `manifest.yml` file.
+      #
+      # @yield [manifest]
+      #
+      # @yieldparam [Hash] manifest
+      #
+      def update_manifest_file(&block)
+        manifest = YAML::Store.new(@manifest_file)
+        manifest.transaction(&block)
+      end
+
+    end
+  end
+end

data/lib/ronin/wordlists/cli/command.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+#
+# ronin-wordlists - A library and tool for managing wordlists.
+#
+# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
+#
+# ronin-wordlists is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-wordlists is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-wordlists.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/wordlists/root'
+
+require 'ronin/core/cli/command'
+
+module Ronin
+  module Wordlists
+    class CLI
+      #
+      # Base class for all `ronin-wordlists` commands.
+      #
+      class Command < Core::CLI::Command
+
+        man_dir File.join(ROOT,'man')
+
+        bug_report_url 'https://github.com/ronin-rb/ronin-wordlists/issues/new'
+
+      end
+    end
+  end
+end

data/lib/ronin/wordlists/cli/commands/completion.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+#
+# ronin-wordlists - A library and tool for managing wordlists.
+#
+# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
+#
+# ronin-wordlists is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-wordlists is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-wordlists.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/wordlists/root'
+require 'ronin/core/cli/completion_command'
+
+module Ronin
+  module Wordlists
+    class CLI
+      module Commands
+        #
+        # Manages the shell completion rules for `ronin-wordlists`.
+        #
+        # ## Usage
+        #
+        #     ronin-wordlists completion [options]
+        #
+        # ## Options
+        #
+        #         --print                      Prints the shell completion file
+        #         --install                    Installs the shell completion file
+        #         --uninstall                  Uninstalls the shell completion file
+        #     -h, --help                       Print help information
+        #
+        # ## Examples
+        #
+        #     ronin-wordlists completion --print
+        #     ronin-wordlists completion --install
+        #     ronin-wordlists completion --uninstall
+        #
+        class Completion < Core::CLI::CompletionCommand
+
+          completion_file File.join(ROOT,'data','completions','ronin-wordlists')
+
+          man_dir File.join(ROOT,'man')
+          man_page 'ronin-wordlists-completion.1'
+
+          description 'Manages the shell completion rules for ronin-wordlists'
+
+        end
+      end
+    end
+  end
+end