wordlist 0.1.1 → 1.0.1

data/lib/wordlist/file.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+require 'wordlist/abstract_wordlist'
+require 'wordlist/exceptions'
+require 'wordlist/format'
+require 'wordlist/compression/reader'
+
+module Wordlist
+  #
+  # Represents a `.txt` file wordlist.
+  #
+  #     wordlist = Wordlist::File.new("rockyou.txt")
+  #     wordlist.each do |word|
+  #       puts word
+  #     end
+  #
+  # @api public
+  #
+  # @since 1.0.0
+  #
+  class File < AbstractWordlist
+
+    # The path to the `.txt` file
+    attr_reader :path
+
+    # The format of the wordlist file.
+    #
+    # @return [:txt, :gzip, :bzip2, :xz]
+    attr_reader :format
+
+    #
+    # Opens a wordlist file.
+    #
+    # @param [String] path
+    #   The path to the `.txt` file wordlist read from.
+    #
+    # @param [:txt, :gz, :bzip2, :xz, nil] format
+    #   The format of the wordlist. If not given the format will be inferred
+    #   from the file extension.
+    #
+    # @raise [WordlistNotFound]
+    #   The given path does not exist.
+    #
+    # @raise [UnknownFormat]
+    #   The format could not be inferred from the file extension.
+    #
+    # @api public
+    #
+    def initialize(path, format: Format.infer(path))
+      @path   = ::File.expand_path(path)
+      @format = format
+
+      unless ::File.file?(@path)
+        raise(WordlistNotFound,"wordlist file does not exist: #{@path.inspect}")
+      end
+
+      unless Format::FORMATS.include?(@format)
+        raise(UnknownFormat,"unknown format given: #{@format.inspect}")
+      end
+    end
+
+    #
+    # Opens a wordlist file.
+    #
+    # @param [String] path
+    #   The path to the `.txt` file wordlist read from.
+    #
+    # @yield [wordlist]
+    #   If a block is given, it will be passed the opened wordlist.
+    #
+    # @yieldparam [File] wordlist
+    #   The newly opened wordlist.
+    #
+    # @return [File]
+    #   The newly opened wordlist.
+    #
+    # @see #initialize
+    #
+    # @api public
+    #
+    def self.open(path,**kwargs)
+      wordlist = new(path,**kwargs)
+      yield wordlist if block_given?
+      return wordlist
+    end
+
+    #
+    # Opens and reads the wordlist file.
+    #
+    # @param [String] path
+    #   The path to the `.txt` file wordlist read from.
+    #
+    # @yield [word]
+    #   The given block will be passed every word from the wordlist.
+    #
+    # @yieldparam [String] word
+    #   A word from the wordlist.
+    #
+    # @return [Enumerator]
+    #   If no block is given, an Enumerator object will be returned.
+    #
+    def self.read(path,**kwargs,&block)
+      open(path,**kwargs).each(&block)
+    end
+
+    #
+    # Enumerates through each line in the `.txt` file wordlist.
+    #
+    # @yield [line]
+    #   The given block will be passed each line from the `.txt` file.
+    #
+    # @yieldparam [String] line
+    #   A newline terminated line from the file.
+    #
+    # @return [Enumerator]
+    #   If no block is given, an Enumerator object will be returned.
+    #
+    # @api semipublic
+    #
+    def each_line(&block)
+      return enum_for(__method__) unless block
+
+      open { |io| io.each_line(&block) }
+    end
+
+    #
+    # Enumerates through every word in the `.txt` file.
+    #
+    # @yield [word]
+    #   The given block will be passed every word from the wordlist.
+    #
+    # @yieldparam [String] word
+    #   A word from the wordlist.
+    #
+    # @return [Enumerator]
+    #   If no block is given, an Enumerator object will be returned.
+    #
+    # @note
+    #   Empty lines and lines beginning with `#` characters will be ignored.
+    #
+    # @example
+    #   wordlist.each do |word|
+    #     puts word
+    #   end
+    #
+    # @api public
+    #
+    def each
+      return enum_for(__method__) unless block_given?
+
+      each_line do |line|
+        line.chomp!
+
+        unless (line.empty? || line.start_with?('#'))
+          yield line
+        end
+      end
+    end
+
+    private
+
+    #
+    # Opens the wordlist for reading.
+    #
+    # @yield [io]
+    #
+    # @yieldparam [IO] io
+    #
+    def open(&block)
+      if @format == :txt
+        ::File.open(@path,&block)
+      else
+        Compression::Reader.open(@path, format: @format, &block)
+      end
+    end
+
+  end
+end

data/lib/wordlist/format.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+require 'wordlist/exceptions'
+
+module Wordlist
+  #
+  # Handles wordlist format detection.
+  #
+  # @since 1.0.0
+  #
+  module Format
+    # Mapping of file extensions to formats
+    FILE_FORMATS = {
+      '.txt' => :txt,
+      '.gz'  => :gzip,
+      '.bz2' => :bzip2,
+      '.xz'  => :xz
+    }
+
+    # Valid formats.
+    FORMATS = FILE_FORMATS.values
+
+    #
+    # Infers the format from the given file name.
+    #
+    # @param [String] path
+    #   The path to the file.
+    #
+    # @return [:txt, :gzip, :bzip2, :xz]
+    #
+    # @raise [UnknownFormat]
+    #   The format could not be inferred from the file path.
+    #
+    def self.infer(path)
+      FILE_FORMATS.fetch(::File.extname(path)) do
+        raise(UnknownFormat,"could not infer the format of file: #{path.inspect}")
+      end
+    end
+  end
+end

data/lib/wordlist/lexer/lang.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wordlist
+  class Lexer
+    #
+    # Detects the system's default language.
+    #
+    # @api semipublic
+    #
+    # @since 1.0.0
+    #
+    module Lang
+      #
+      # The default language.
+      #
+      # @return [Symbol]
+      #
+      def self.default
+        if (lang = ENV['LANG'])
+          lang, encoding = lang.split('.',2)
+          lang, country = lang.split('_',2)
+
+          unless lang == 'C'
+            lang.to_sym
+          else
+            :en
+          end
+        else
+          :en
+        end
+      end
+    end
+  end
+end

data/lib/wordlist/lexer/stop_words.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+require 'wordlist/exceptions'
+
+module Wordlist
+  class Lexer
+    #
+    # Stop words for various languages.
+    #
+    # @api semipublic
+    #
+    # @since 1.0.0
+    #
+    module StopWords
+      # The directory containing the stop words `.txt` files.
+      DIRECTORY = ::File.expand_path(::File.join(__dir__,'..','..','..','data','stop_words'))
+
+      #
+      # The path to the stop words `.txt` file.
+      #
+      # @param [Symbol] lang
+      #   The language to load.
+      #
+      # @return [String]
+      #
+      def self.path_for(lang)
+        ::File.join(DIRECTORY,"#{lang}.txt")
+      end
+
+      #
+      # Reads the stop words.
+      #
+      # @param [Symbol] lang
+      #   The language to load.
+      #
+      # @return [Array<String>]
+      #
+      # @raise [UnsupportedLanguage]
+      #
+      def self.read(lang)
+        path = path_for(lang)
+
+        unless ::File.file?(path)
+          raise(UnsupportedLanguage,"unsupported language: #{lang}")
+        end
+
+        lines = ::File.readlines(path)
+        lines.each(&:chomp!)
+        lines
+      end
+
+      @stop_words = {}
+      @mutex = Mutex.new
+
+      #
+      # Lazy loads the stop words for the given language.
+      #
+      # @param [Symbol] lang
+      #   The language to load.
+      #
+      # @return [Array<String>]
+      #
+      def self.[](lang)
+        @mutex.synchronize do
+          @stop_words[lang] ||= read(lang)
+        end
+      end
+    end
+  end
+end

data/lib/wordlist/lexer.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+require 'wordlist/lexer/lang'
+require 'wordlist/lexer/stop_words'
+
+require 'strscan'
+
+module Wordlist
+  #
+  # Parses arbitrary text and scans each word from it.
+  #
+  # @api semipublic
+  #
+  # @since 1.0.0
+  #
+  class Lexer
+
+    # Regexp to match acronyms.
+    ACRONYM = /[[:alpha:]](?:\.[[:alpha:]])+\./
+
+    # Default set of punctuation characters allowed within words
+    SPECIAL_CHARS = %w[_ - ']
+
+    # @return [Symbol]
+    attr_reader :lang
+
+    # @return [Array<String>]
+    attr_reader :stop_words
+
+    # @return [Array<String, Regexp>]
+    attr_reader :ignore_words
+
+    # @return [Array<String>]
+    attr_reader :special_chars
+
+    #
+    # Initializes the lexer.
+    #
+    # @param [Symbol] lang
+    #   The language to use. Defaults to {Lang.default}.
+    #
+    # @param [Array<String>] stop_words
+    #   The explicit stop-words to ignore. If not given, default stop words
+    #   will be loaded based on `lang` or {Lang.default}.
+    #
+    # @param [Array<String, Regexp>] ignore_words
+    #   Optional list of words to ignore. Can contain Strings or Regexps.
+    #
+    # @param [Boolean] digits
+    #   Controls whether parsed words may contain digits or not.
+    #
+    # @param [Array<String>] special_chars
+    #   The additional special characters allowed within words.
+    #
+    # @param [Boolean] numbers
+    #   Controls whether whole numbers will be parsed as words.
+    #
+    # @param [Boolean] acronyms
+    #   Controls whether acronyms will be parsed as words.
+    #
+    # @param [Boolean] normalize_case
+    #   Controls whether to convert all words to lowercase.
+    #
+    # @param [Boolean] normalize_apostrophes
+    #   Controls whether apostrophes will be removed from the end of words.
+    #
+    # @param [Boolean] normalize_acronyms
+    #   Controls whether acronyms will have `.` characters removed.
+    #
+    # @raise [ArgumentError]
+    #   The `ignore_words` keyword contained a value other than a String or
+    #   Regexp.
+    #
+    def initialize(lang:          Lang.default,
+                   stop_words:    StopWords[lang],
+                   ignore_words:  [],
+                   digits:   true,
+                   special_chars:  SPECIAL_CHARS,
+                   numbers:  false,
+                   acronyms: true,
+                   normalize_case:        false,
+                   normalize_apostrophes: false,
+                   normalize_acronyms:    false)
+      @lang          = lang
+      @stop_words    = stop_words
+      @ignore_words  = ignore_words
+      @special_chars = special_chars
+
+      @digits   = digits
+      @numbers  = numbers
+      @acronyms = acronyms
+
+      @normalize_acronyms    = normalize_acronyms
+      @normalize_apostrophes = normalize_apostrophes
+      @normalize_case        = normalize_case
+
+      escaped_chars = Regexp.escape(@special_chars.join)
+
+      @word = if @digits
+                # allows numeric characters
+                /\p{L}(?:[\p{L}\p{Nd}#{escaped_chars}]*[\p{L}\p{Nd}])?/
+              else
+                # only allows alpha characters
+                /\p{L}(?:[\p{L}#{escaped_chars}]*\p{L})?/
+              end
+
+      skip_words = Regexp.union(
+        (@stop_words + @ignore_words).map { |pattern|
+          case pattern
+          when Regexp then pattern
+          when String then /#{Regexp.escape(pattern)}/i
+          else
+            raise(ArgumentError,"ignore_words: must contain only Strings or Regexps")
+          end
+        }
+      )
+
+      if @numbers
+        # allows lexing whole numbers
+        @skip_word   = /(?:#{skip_words}[[:punct:]]*(?:[[:space:]]+|$))+/i
+        @word        = /#{@word}|\d+/
+        @not_a_word  = /[^\p{L}\d]+/
+      else
+        # skips whole numbers
+        @skip_word   = /(?:(?:#{skip_words}|\d+)[[:punct:]]*(?:[[:space:]]+|$))+/i
+        @not_a_word  = /[^\p{L}]+/
+      end
+    end
+
+    #
+    # Determines whether parsed words may contain digits or not.
+    #
+    # @return [Boolean]
+    #
+    def digits?
+      @digits
+    end
+
+    #
+    # Determines whether numbers will be parsed or ignored.
+    #
+    # @return [Boolean]
+    #
+    def numbers?
+      @numbers
+    end
+
+    #
+    # Determines whether acronyms will be parsed or ignored.
+    #
+    # @return [Boolean]
+    #
+    def acronyms?
+      @acronyms
+    end
+
+    #
+    # Determines whether `.` characters will be removed from acronyms.
+    #
+    # @return [Boolean]
+    #
+    def normalize_acronyms?
+      @normalize_acronyms
+    end
+
+    #
+    # Determines whether apostrophes will be stripped from words.
+    #
+    # @return [Boolean]
+    #
+    def normalize_apostrophes?
+      @normalize_apostrophes
+    end
+
+    #
+    # Determines whether all words will be converted to lowercase.
+    #
+    # @return [Boolean]
+    #
+    def normalize_case?
+      @normalize_case
+    end
+
+    #
+    # Enumerates over each word in the text.
+    #
+    # @yield [word]
+    #   The given block will be passed each word from the text.
+    #
+    # @yieldparam [String] word
+    #   A parsed word from the text.
+    #
+    # @return [Array<String>]
+    #   If no block is given, an Array of the parsed words will be returned
+    #   instead.
+    #
+    def parse(text,&block)
+      return enum_for(__method__,text).to_a unless block_given?
+
+      scanner = StringScanner.new(text)
+
+      until scanner.eos?
+        scanner.skip(@not_a_word)
+        scanner.skip(@skip_word)
+
+        if (acronym = scanner.scan(ACRONYM))
+          if @acronyms
+            acronym.tr!('.','') if @normalize_acronyms
+
+            yield acronym
+          end
+        elsif (word = scanner.scan(@word))
+          word.downcase! if @normalize_case
+          word.chomp!("'s") if (@normalize_apostrophes && word.end_with?("'s"))
+
+          yield word
+        end
+      end
+    end
+
+  end
+end