wordlist 0.1.1 → 1.0.0

metadata

@@ -1,167 +1,230 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: wordlist
-version: !ruby/object:Gem::Version 
-  hash: 25
-  prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 1
-  version: 0.1.1
+version: !ruby/object:Gem::Version
+  version: 1.0.0
 platform: ruby
-authors: 
+authors:
 - Postmodern
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2012-06-12 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: spidr
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 0
-        - 2
-        version: "0.2"
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+date: 2021-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rubygems-tasks
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 9
-        segments: 
-        - 0
-        - 1
-        version: "0.1"
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 4
-        version: "2.4"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: yard
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 0
-        - 8
-        version: "0.8"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
   type: :development
-  version_requirements: *id004
-description: A Ruby library for generating and working with word-lists. Wordlist allows one to efficiently generate unique word-lists from arbitrary text or other sources, such as website content. Wordlist can also quickly enumerate through words within an existing word-list, applying multiple mutation rules to each word in the list.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Wordlist is a Ruby library for reading, manipulating, and creating wordlists,
+  efficiently.
 email: postmodern.mod3@gmail.com
-executables: 
+executables:
 - wordlist
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - ChangeLog.md
 - LICENSE.txt
 - README.md
-files: 
-- .document
-- .gitignore
-- .rspec
-- .yardopts
+files:
+- ".document"
+- ".github/workflows/ruby.yml"
+- ".gitignore"
+- ".rspec"
+- ".yardopts"
 - ChangeLog.md
+- Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmarks.rb
 - bin/wordlist
+- data/stop_words/ar.txt
+- data/stop_words/bg.txt
+- data/stop_words/bn.txt
+- data/stop_words/ca.txt
+- data/stop_words/cs.txt
+- data/stop_words/da.txt
+- data/stop_words/de.txt
+- data/stop_words/el.txt
+- data/stop_words/en.txt
+- data/stop_words/es.txt
+- data/stop_words/eu.txt
+- data/stop_words/fa.txt
+- data/stop_words/fi.txt
+- data/stop_words/fr.txt
+- data/stop_words/ga.txt
+- data/stop_words/gl.txt
+- data/stop_words/he.txt
+- data/stop_words/hi.txt
+- data/stop_words/hr.txt
+- data/stop_words/hu.txt
+- data/stop_words/hy.txt
+- data/stop_words/id.txt
+- data/stop_words/it.txt
+- data/stop_words/ja.txt
+- data/stop_words/ko.txt
+- data/stop_words/ku.txt
+- data/stop_words/lt.txt
+- data/stop_words/lv.txt
+- data/stop_words/mr.txt
+- data/stop_words/nl.txt
+- data/stop_words/no.txt
+- data/stop_words/pl.txt
+- data/stop_words/pt.txt
+- data/stop_words/ro.txt
+- data/stop_words/ru.txt
+- data/stop_words/sk.txt
+- data/stop_words/sv.txt
+- data/stop_words/th.txt
+- data/stop_words/tr.txt
+- data/stop_words/uk.txt
+- data/stop_words/ur.txt
+- data/stop_words/zh.txt
 - gemspec.yml
 - lib/wordlist.rb
+- lib/wordlist/abstract_wordlist.rb
 - lib/wordlist/builder.rb
-- lib/wordlist/builders.rb
-- lib/wordlist/builders/website.rb
-- lib/wordlist/flat_file.rb
-- lib/wordlist/list.rb
-- lib/wordlist/mutator.rb
-- lib/wordlist/parsers.rb
-- lib/wordlist/runners.rb
-- lib/wordlist/runners/list.rb
-- lib/wordlist/runners/runner.rb
+- lib/wordlist/cli.rb
+- lib/wordlist/compression/reader.rb
+- lib/wordlist/compression/writer.rb
+- lib/wordlist/exceptions.rb
+- lib/wordlist/file.rb
+- lib/wordlist/format.rb
+- lib/wordlist/lexer.rb
+- lib/wordlist/lexer/lang.rb
+- lib/wordlist/lexer/stop_words.rb
+- lib/wordlist/list_methods.rb
+- lib/wordlist/modifiers.rb
+- lib/wordlist/modifiers/capitalize.rb
+- lib/wordlist/modifiers/downcase.rb
+- lib/wordlist/modifiers/gsub.rb
+- lib/wordlist/modifiers/modifier.rb
+- lib/wordlist/modifiers/mutate.rb
+- lib/wordlist/modifiers/mutate_case.rb
+- lib/wordlist/modifiers/sub.rb
+- lib/wordlist/modifiers/tr.rb
+- lib/wordlist/modifiers/upcase.rb
+- lib/wordlist/operators.rb
+- lib/wordlist/operators/binary_operator.rb
+- lib/wordlist/operators/concat.rb
+- lib/wordlist/operators/intersect.rb
+- lib/wordlist/operators/operator.rb
+- lib/wordlist/operators/power.rb
+- lib/wordlist/operators/product.rb
+- lib/wordlist/operators/subtract.rb
+- lib/wordlist/operators/unary_operator.rb
+- lib/wordlist/operators/union.rb
+- lib/wordlist/operators/unique.rb
 - lib/wordlist/unique_filter.rb
 - lib/wordlist/version.rb
-- scripts/benchmark
-- scripts/text/comedy_of_errors.txt
+- lib/wordlist/words.rb
+- spec/abstract_list_spec.rb
 - spec/builder_examples.rb
 - spec/builder_spec.rb
 - spec/classes/parser_class.rb
 - spec/classes/test_list.rb
-- spec/flat_file_spec.rb
+- spec/cli_spec.rb
+- spec/compression/reader_spec.rb
+- spec/compression/writer_spec.rb
+- spec/file_spec.rb
+- spec/fixtures/wordlist.txt
+- spec/fixtures/wordlist.txt.bz2
+- spec/fixtures/wordlist.txt.gz
+- spec/fixtures/wordlist.txt.xz
+- spec/fixtures/wordlist_with_ambiguous_format
+- spec/fixtures/wordlist_with_comments.txt
+- spec/fixtures/wordlist_with_empty_lines.txt
+- spec/format_spec.rb
 - spec/helpers/text.rb
 - spec/helpers/wordlist.rb
-- spec/list_spec.rb
-- spec/mutator_spec.rb
-- spec/parsers_spec.rb
+- spec/lexer/lang_spec.rb
+- spec/lexer/stop_words_spec.rb
+- spec/lexer_spec.rb
+- spec/list_methods_spec.rb
+- spec/modifiers/capitalize_spec.rb
+- spec/modifiers/downcase_spec.rb
+- spec/modifiers/gsub_spec.rb
+- spec/modifiers/modifier_spec.rb
+- spec/modifiers/mutate_case_spec.rb
+- spec/modifiers/mutate_spec.rb
+- spec/modifiers/sub_spec.rb
+- spec/modifiers/tr_spec.rb
+- spec/modifiers/upcase_spec.rb
+- spec/operators/binary_operator_spec.rb
+- spec/operators/concat_spec.rb
+- spec/operators/intersect_spec.rb
+- spec/operators/operator_spec.rb
+- spec/operators/power_spec.rb
+- spec/operators/product_spec.rb
+- spec/operators/subtract_spec.rb
+- spec/operators/union_spec.rb
+- spec/operators/unique_spec.rb
 - spec/spec_helper.rb
 - spec/text/flat_file.txt
 - spec/text/previous_wordlist.txt
 - spec/text/sample.txt
 - spec/unique_filter_spec.rb
 - spec/wordlist_spec.rb
+- spec/words_spec.rb
 - wordlist.gemspec
-homepage: https://github.com/sophsec/wordlist
-licenses: 
+homepage: https://github.com/postmodern/wordlist.rb
+licenses:
 - MIT
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.8.23
-signing_key: 
-specification_version: 3
-summary: A Ruby library for generating and working with word-lists.
+rubygems_version: 3.2.22
+signing_key:
+specification_version: 4
+summary: Ruby library for reading, manipulating, and creating wordlists.
 test_files: []
-

data/lib/wordlist/builders/website.rb

@@ -1,216 +0,0 @@
-require 'wordlist/builder'
-
-require 'spidr'
-
-module Wordlist
-  module Builders
-    class Website < Builder
-
-      # Proxy to use
-      attr_accessor :proxy
-
-      # User-Agent to use
-      attr_accessor :user_agent
-
-      # Referer URL to use
-      attr_accessor :referer
-
-      # Host to spider
-      attr_accessor :host
-
-      # HTTP Host Header to use in all requests.
-      attr_accessor :host_header
-
-      # Additional hosts that can be spidered
-      attr_reader :hosts
-
-      # Links to ignore while spidering
-      attr_reader :ignore_links
-
-      # Specifies whether the `content` attribute of `meta` tags will be
-      # parsed
-      attr_accessor :parse_meta
-
-      # Specifies whether `title` tags will be parsed
-      attr_accessor :parse_title
-
-      # Specifies whether `h1` tags will be parsed
-      attr_accessor :parse_h1
-
-      # Specifies whether `h2` tags will be parsed
-      attr_accessor :parse_h2
-
-      # Specifies whether `h3` tags will be parsed
-      attr_accessor :parse_h3
-
-      # Specifies whether `h4` tags will be parsed
-      attr_accessor :parse_h4
-
-      # Specifies whether `h5` tags will be parsed
-      attr_accessor :parse_h5
-
-      # Specifies whether `p` tags will be parsed
-      attr_accessor :parse_p
-
-      # Specifies whether `span` tags will be parsed
-      attr_accessor :parse_span
-
-      # Specifies whether the `alt` attributes of `img` tags will be parsed
-      attr_accessor :parse_alt
-
-      # Specifies whether HTML comment tags will be parsed
-      attr_accessor :parse_comments
-
-      # Additional XPath expressions to use to parse spidered pages
-      attr_reader :xpaths
-
-      #
-      # Creates a new Website builder object.
-      #
-      # @param [String] path
-      #   The path to the word-list to build.
-      #
-      # @param [Hash] options
-      #   Additional options.
-      #
-      # @option options [Hash] :proxy
-      #   The Hash of proxy information to use.
-      #
-      # @option options [String] :user_agent
-      #   The User-Agent string to send with each request.
-      #
-      # @option options [String] :referer
-      #   The Referer URL to send with each request.
-      #
-      # @option options [String] :host_header
-      #   The HTTP Host header to use in all requests.
-      #
-      # @option options [Array<String, Regexp, Proc>] :ignore_links
-      #   Links to ignore while spidering.
-      #
-      # @option options [Boolean] :parse_meta (true)
-      #   Specifies whether the `content` attribute of `meta` tags will be
-      #   parsed.
-      #
-      # @option options [Boolean] :parse_title (true)
-      #   Specifies whether `title` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_h1 (true)
-      #   Specifies whether `h1` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_h2 (true)
-      #   Specifies whether `h2` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_h3 (true)
-      #   Specifies whether `h3` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_h4 (true)
-      #   Specifies whether `h4` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_h5 (true)
-      #   Specifies whether `h5` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_p (true)
-      #   Specifies whether `p` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_span (true)
-      #   Specifies whether `span` tags will be parsed.
-      #
-      # @option options [Boolean] :parse_alt (true)
-      #   Specifies whether the `alt` attributes of `img` tags will be
-      #   parsed.
-      #
-      # @option options [Boolean] :parse_comments (false)
-      #   Specifies whether HTML comment tags will be parsed.
-      #
-      # @option options [Array<String>] :xpaths
-      #   Additional list of XPath expressions, to use when parsing
-      #   spidered pages.
-      #
-      def initialize(path,options={},&block)
-        @proxy      = options.fetch(:proxy,Spidr.proxy)
-        @user_agent = options[:user_agent]
-        @referer    = options[:referer]
-
-        @host        = options[:host]
-        @host_header = options[:host_header]
-        @hosts       = Array(options[:hosts])
-
-        @ignore_links = Array(options[:ignore_links])
-
-        @parse_meta     = options.fetch(:parse_meta,true)
-        @parse_title    = options.fetch(:parse_title,true)
-        @parse_h1       = options.fetch(:parse_h1,true)
-        @parse_h2       = options.fetch(:parse_h2,true)
-        @parse_h3       = options.fetch(:parse_h3,true)
-        @parse_h4       = options.fetch(:parse_h4,true)
-        @parse_h5       = options.fetch(:parse_h5,true)
-        @parse_p        = options.fetch(:parse_p,true)
-        @parse_span     = options.fetch(:parse_span,true)
-        @parse_alt      = options.fetch(:parse_alt,true)
-        @parse_comments = options.fetch(:parse_comments,false)
-
-        @xpaths = Array(options[:xpaths])
-
-        super(path,options,&block)
-      end
-
-      #
-      # Builds the word-list file by spidering the `host` and parsing the
-      # inner-text from all HTML pages.
-      #
-      # @yield [builder]
-      #   If a block is given, it will be called before all HTML pages on
-      #   the `host` have been parsed.
-      #
-      # @yieldparam [Website] builder
-      #   The website word-list builder.
-      #
-      def build!(&block)
-        super(&block)
-
-        options = {
-          :proxy        => @proxy,
-          :user_agent   => @user_agent,
-          :referer      => @referer,
-          :hosts        => @hosts,
-          :ignore_links => @ignore_links
-        }
-
-        xpaths = []
-        xpaths << '//meta/@content' if @parse_meta
-        xpaths << '//title'         if @parse_title
-        xpaths << '//h1'            if @parse_h1
-        xpaths << '//h2'            if @parse_h2
-        xpaths << '//h3'            if @parse_h3
-        xpaths << '//h4'            if @parse_h4
-        xpaths << '//h5'            if @parse_h5
-        xpaths << '//p'             if @parse_p
-        xpaths << '//span'          if @parse_span
-        xpaths << '//img/@alt'      if @parse_alt
-        xpaths += @xpaths
-
-        Spidr.host(@host,options) do |spidr|
-          spidr.every_page do |page|
-            if page.html?
-              if page.doc
-                xpaths.each do |xpath|
-                  page.doc.search(xpath).each do |element|
-                    parse(element.inner_text)
-                  end
-                end
-              end
-
-              if (@parse_comments && page.doc)
-                page.doc.traverse do |element|
-                  parse(element.inner_text) if element.comment?
-                end
-              end
-            end
-          end
-        end
-      end
-
-    end
-  end
-end

data/lib/wordlist/builders.rb

@@ -1 +0,0 @@
-require 'wordlist/builders/website'

data/lib/wordlist/flat_file.rb

@@ -1,47 +0,0 @@
-require 'wordlist/list'
-
-module Wordlist
-  class FlatFile < List
-
-    # The path to the flat-file
-    attr_accessor :path
-
-    #
-    # Opens a new FlatFile list.
-    #
-    # @param [String] path
-    #   The path to the flat file word-list read from.
-    #
-    # @param [Hash] options
-    #   Additional options.
-    #
-    def initialize(path,options={},&block)
-      @path = path
-
-      super(options,&block)
-    end
-
-    #
-    # Enumerates through every word in the flat-file.
-    #
-    # @yield [word]
-    #   The given block will be passed every word from the word-list.
-    #
-    # @yieldparam [String] word
-    #   A word from the word-list.
-    #
-    # @example
-    #   flat_file.each_word do |word|
-    #     puts word
-    #   end
-    #
-    def each_word(&block)
-      File.open(@path) do |file|
-        file.each_line do |line|
-          yield line.chomp
-        end
-      end
-    end
-
-  end
-end