wordlist 0.1.1 → 1.0.0

data/lib/wordlist/operators/subtract.rb

@@ -0,0 +1,54 @@
+require 'wordlist/operators/binary_operator'
+require 'wordlist/unique_filter'
+
+module Wordlist
+  module Operators
+    #
+    # Lazily enumerates over every word in the first wordlist, that is not in
+    # the second wordlist.
+    #
+    # @since 1.0.0
+    #
+    class Subtract < BinaryOperator
+
+      #
+      # Enumerates over the difference between the two wordlists.
+      #
+      # @yield [word]
+      #   The given block will be passed each word from the first wordlist,
+      #   that is not in the second wordlist.
+      #
+      # @yieldparam [String] word
+      #   A word that belongs to first wordlist, but not the second wordlist.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enumerator object will be returned.
+      #
+      # @example
+      #   wordlist1 = Wordlist::List["foo", "bar", baz", "qux"]
+      #   wordlist2 = Wordlist::List["bar", "qux"]
+      #   (wordlist1 - wordlist2).each do |word|
+      #     puts word
+      #   end
+      #   # foo
+      #   # baz
+      #
+      # @api public
+      #
+      def each
+        return enum_for(__method__) unless block_given?
+
+        unique_filter = UniqueFilter.new
+
+        @right.each { |word| unique_filter.add(word) }
+
+        @left.each do |word|
+          unless unique_filter.include?(word)
+            yield word
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wordlist/operators/unary_operator.rb

@@ -0,0 +1,29 @@
+require 'wordlist/operators/operator'
+
+module Wordlist
+  module Operators
+    #
+    # Unary operator base class.
+    #
+    # @since 1.0.0
+    #
+    class UnaryOperator < Operator
+
+      # The wordlist to operate on.
+      #
+      # @return [Enumerable]
+      attr_reader :wordlist
+
+      #
+      # Initializes the unary operator.
+      #
+      # @param [Enumerable] wordlist
+      #   The wordlist.
+      #
+      def initialize(wordlist)
+        @wordlist = wordlist
+      end
+
+    end
+  end
+end

data/lib/wordlist/operators/union.rb

@@ -0,0 +1,61 @@
+require 'wordlist/operators/binary_operator'
+require 'wordlist/unique_filter'
+
+module Wordlist
+  module Operators
+    #
+    # Lazily enumerates over words from both wordlists, filtering out any
+    # duplicates.
+    #
+    # @since 1.0.0
+    #
+    class Union < BinaryOperator
+
+      #
+      # Enumerates over the union of the two wordlists.
+      #
+      # @yield [word]
+      #   The given block will be passed each word from both wordlists,
+      #   without duplicates.
+      #
+      # @yieldparam [String] word
+      #   A word that belongs to one of the wordlists.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enumerator object will be returned.
+      #
+      # @example
+      #   wordlist1 = Wordlist::List["foo", "bar", "baz", "qux"]
+      #   wordlist2 = Wordlist::List["xyz", "bar", "abc", "qux"]
+      #   (wordlist1 | wordlist2).each do |word|
+      #     puts word
+      #   end
+      #   # foo
+      #   # bar
+      #   # baz
+      #   # qux
+      #   # xyz
+      #   # abc
+      #
+      # @api public
+      #
+      def each
+        return enum_for(__method__) unless block_given?
+
+        unique_filter = UniqueFilter.new
+
+        @left.each do |word|
+          yield word
+          unique_filter.add(word)
+        end
+
+        @right.each do |word|
+          unless unique_filter.include?(word)
+            yield word
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wordlist/operators/unique.rb

@@ -0,0 +1,52 @@
+require 'wordlist/operators/unary_operator'
+require 'wordlist/unique_filter'
+
+module Wordlist
+  module Operators
+    #
+    # Lazily enumerates over only the unique words in the wordlist, filtering
+    # out duplicates.
+    #
+    # @since 1.0.0
+    #
+    class Unique < UnaryOperator
+
+      #
+      # Enumerates over the unique words in the wordlist.
+      #
+      # @yield [word]
+      #   The given block will be passed each unique word from the wordlist.
+      #
+      # @yieldparam [String] word
+      #   A unique word from the wordlist.
+      #
+      # @return [Enumerator]
+      #   If no block is given, an Enumerator object will be returned.
+      #
+      # @example
+      #   wordlist= Wordlist::List["foo", "bar", "baz", "qux"]
+      #   (wordlist + wordlist).uniq.each do |word|
+      #     puts word
+      #   end
+      #   # foo
+      #   # bar
+      #   # baz
+      #   # qux
+      #
+      # @api public
+      #
+      def each
+        return enum_for(__method__) unless block_given?
+
+        unique_filter = UniqueFilter.new
+
+        @wordlist.each do |word|
+          if unique_filter.add?(word)
+            yield word
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wordlist/operators.rb

@@ -0,0 +1,7 @@
+require 'wordlist/operators/concat'
+require 'wordlist/operators/subtract'
+require 'wordlist/operators/product'
+require 'wordlist/operators/power'
+require 'wordlist/operators/intersect'
+require 'wordlist/operators/union'
+require 'wordlist/operators/unique'

data/lib/wordlist/unique_filter.rb

@@ -1,16 +1,25 @@
 require 'set'
 
 module Wordlist
+  #
+  # Acts as a filter to filter out duplicate words.
+  #
+  # @api semipublic
+  #
+  # @since 1.0.0
+  #
   class UniqueFilter
 
-    # CRC32 Hashes of words seen so far
-    attr_reader :seen
+    # The seen String hashes
+    #
+    # @return [Set<Integer>]
+    attr_reader :hashes
 
     #
-    # Creates a new UniqueFilter object.
+    # Creates a new unique filter.
     #
     def initialize
-      @seen = {}
+      @hashes = Set.new
     end
 
     #
@@ -22,90 +31,60 @@ module Wordlist
     # @return [Boolean]
     #   Specifies whether the word has been previously seen.
     #
-    def seen?(word)
-      length = word.length
-
-      (@seen.has_key?(length) && @seen[length].include?(crc32(word)))
+    def include?(word)
+      @hashes.include?(word.hash)
     end
 
     #
-    # Marks the given word as previously seen.
+    # Adds the word to the unique filter.
     #
     # @param [String] word
-    #   The word to mark as previously seen.
+    #   The word to add.
     #
-    # @return [Boolean]
-    #   Specifies whether or not the word has not been previously seen
-    #   until now.
-    #
-    def saw!(word)
-      length = word.length
-      crc    = crc32(word)
-
-      if @seen.has_key?(length)
-        return false if @seen[length].include?(crc)
-        @seen[length] << crc
-      else
-        @seen[length] = SortedSet[crc]
-      end
-
-      return true
+    def add(word)
+      @hashes.add(word.hash)
     end
 
+    alias << add
+
     #
-    # Passes the given word through the unique filter.
+    # Attempts to add the word to the unique filter.
     #
     # @param [String] word
-    #   The word to pass through the unique filter.
+    #   The word to add.
     #
-    # @yield [word]
-    #   The given block will be passed the word, if the word has not been
-    #   previously seen by the filter.
+    # @return [Boolean]
+    #   Returns `true` if the word does not yet exist in the unique filter.
+    #   Returns `false` if the word already exists in the unique filter.
     #
-    # @yieldparam [String] word
-    #   A unique word that has not been previously seen by the filter.
+    def add?(word)
+      !@hashes.add?(word.hash).nil?
+    end
+
     #
-    # @return [nil]
+    # Determines if the unique filter is empty or not.
     #
-    def pass(word)
-      if saw!(word)
-        yield word
-      end
-
-      return nil
+    # @return [Boolean]
+    #
+    def empty?
+      @hashes.empty?
     end
 
     #
     # Clears the unique filter.
     #
-    # @return [UniqueFilter]
-    #   The cleared filter.
-    #
     def clear
-      @seen.clear
-      return self
+      @hashes.clear
     end
 
-    protected
-
     #
-    # Returns the CRC32 checksum of the given word.
-    #
-    # @param [String] word
-    #   The word to calculate a CRC32 checksum for.
+    # The size of the unique filter.
     #
     # @return [Integer]
-    #   The CRC32 checksum for the given word.
+    #   The number of unique words seen by the unique filter.
     #
-    def crc32(word)
-      r = 0xffffffff
-
-      word.each_byte do |b|
-        r ^= b
-        8.times { r = ((r >> 1) ^ (0xEDB88320 * (r & 1))) }
-      end
-
-      r ^ 0xffffffff
+    def size
+      @hashes.size
     end
 
   end

data/lib/wordlist/version.rb

@@ -1,4 +1,4 @@
 module Wordlist
   # Word version
-  VERSION = '0.1.1'
+  VERSION = '1.0.0'
 end

data/lib/wordlist/words.rb

@@ -0,0 +1,71 @@
+require "wordlist/abstract_wordlist"
+
+module Wordlist
+  #
+  # An in-memory wordlist of words.
+  #
+  #     Wordlist::Words["foo", "bar", "baz"]
+  #
+  # @api public
+  #
+  # @since 1.0.0
+  #
+  class Words < AbstractWordlist
+
+    # The words in the wordlist.
+    #
+    # @return [Array<String>, Enumerable]
+    attr_reader :words
+
+    #
+    # Creates a new wordlist object.
+    #
+    # @param [Array<String>, Enumerable] words
+    #   The words for the wordlist.
+    #
+    # @api public
+    #
+    def initialize(words=[])
+      @words = words
+    end
+
+    #
+    # Creates a new wordlist from the given words.
+    #
+    # @param [Array<String>] words
+    #   The words for the wordlist.
+    #
+    # @example
+    #   Wordlist::Words["foo", "bar", "baz"]
+    #
+    # @api public
+    #
+    def self.[](*words)
+      new(words)
+    end
+
+    #
+    # Enumerate through every word in the in-memory wordlist.
+    #
+    # @yield [word]
+    #   The given block will be passed each word in the list.
+    #
+    # @yieldparam [String] word
+    #   A word from the in-memory wordlist.
+    #
+    # @return [Enumerator]
+    #   If no block is given, then an `Enumerator` object will be returned.
+    #
+    # @example
+    #   words.each do |word|
+    #     puts word
+    #   end
+    #
+    # @api public
+    #
+    def each(&block)
+      @words.each(&block)
+    end
+
+  end
+end

data/lib/wordlist.rb

@@ -1,4 +1,105 @@
+require 'wordlist/words'
+require 'wordlist/file'
 require 'wordlist/builder'
-require 'wordlist/list'
-require 'wordlist/flat_file'
 require 'wordlist/version'
+
+module Wordlist
+  # 
+  # Creates an in-memory wordlist from the given words.
+  #
+  # @param [Array<String>] words
+  #   The literal words for the list.
+  #
+  # @return [File]
+  #   The in-memory wordlist.
+  #
+  # @api public
+  #
+  # @since 1.0.0
+  #
+  def self.[](*words)
+    Words[*words]
+  end
+
+  #
+  # Opens a wordlist file.
+  #
+  # @param [String] path
+  #   The path to the file.
+  #
+  # @param [Hash{Symbol => Object}] kwargs
+  #   Additional keyword arguments for {Wordlist::File#initialize}.
+  #
+  # @option [:txt, :bzip, :bzip2, :xz] :format
+  #   Specifies the format of the wordlist. If no format is given, the format
+  #   will be inferred from the path's file extension.
+  #
+  # @yield [wordlist]
+  #   If a block is given, it will be passed the newly opened wordlist.
+  #
+  # @yieldparam [Wordlist::File] wordlist
+  #   The newly opened wordlist.
+  #
+  # @return [Wordlist::File]
+  #   The opened wordlist.
+  #
+  # @raise [ArgumentError]
+  #   No `format:` was given, the wordlist format could not be inferred from the
+  #   path's file extension.
+  #
+  # @api public
+  #
+  # @since 1.0.0
+  #
+  def self.open(path,**kwargs,&block)
+    File.open(path,**kwargs,&block)
+  end
+
+  #
+  # Creates a new wordlist builder.
+  #
+  # @param [String] path
+  #   The path to the file.
+  #
+  # @param [Hash{Symbol => Object}] kwargs
+  #   Additional keyword arguments for {Builder#initialize}.
+  #
+  # @option kwargs [:txt, :bzip, :bzip2, :xz, nil] :format
+  #   Specifies the format of the wordlist. If no format is given, the format
+  #   will be inferred from the path's file extension.
+  #
+  # @option kwargs [Boolean] :append
+  #   Specifies whether new words will be appended onto the end of the wordlist
+  #   file or if it will be overwritten.
+  #
+  # @yield [builder]
+  #   If a block is given, the newly created builder object will be yielded.
+  #   After the block has returned the builder will automatically be closed.
+  #
+  # @yieldparam [Builder] builder
+  #   The newly created builder object.
+  #
+  # @return [Builder]
+  #
+  # @raise [ArgumentError]
+  #   No `format:` was given, the wordlist format could not be inferred from the
+  #   path's file extension.
+  #
+  # @api public
+  #
+  # @since 1.0.0
+  #
+  def self.build(path,**kwargs)
+    builder = Builder.new(path,**kwargs)
+
+    if block_given?
+      begin
+        yield builder
+      ensure
+        builder.close
+      end
+    end
+
+    return builder
+  end
+end

data/spec/abstract_list_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+require 'wordlist/abstract_wordlist'
+
+describe Wordlist::AbstractWordlist do
+  it do
+    expect(described_class).to include(Enumerable)
+  end
+
+  it do
+    expect(described_class).to include(Wordlist::ListMethods)
+  end
+
+  describe "#each" do
+    it do
+      expect { subject.each }.to raise_error(NotImplementedError,"#{described_class}#each was not implemented")
+    end
+  end
+end