logfile_interval 2.1.5 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2744d333999683078bcd1e6251df9f4a77592ba
-  data.tar.gz: 39977126f145556de0e49a5bcd75fae80c862dc3
+  metadata.gz: c7f2e7cabde319d9e316d67b25f41786d30a3a65
+  data.tar.gz: 31af6fa687dbd9368e13faa64154ce778d27f3a2
 SHA512:
-  metadata.gz: 9e23e0e91fb21f9fb39e40fc9ca002fbafec454914de10f98951f42b39c9c8e19339d8db508ae86d6489c4aff810f118952af673b1e5cd799f8783c44bb82cf9
-  data.tar.gz: f3e73d0ddd1d07611bfe8476a823dcc5eed323d866404ee782f10afc217389e96f5b7ec8109c4781a11dbb06b888f2d4ebda9050cbf82f0c8a89c000399c6d4e
+  metadata.gz: bafa4a0527c1c2b33479ce61276830c39ca588648c95a22e39660e3c26ec410dcdeaf9c47453902619347e6f7b15dcdb40d0dfb8ef8a6fb8e2584882a3b77ffc
+  data.tar.gz: 0dcafb906ba7c9ad3e2e88733858e8e54cfa9a54ae8e9f94a3e3acc6d4a14b87c4193541d1af98620bdac73ebd2ad3721815b56fa4ecf282e6bcb93d46be1855

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1.2
+  - 2.4.3
+  - 2.5.0
 script: bundle exec rspec spec

data/README.md

@@ -33,6 +33,8 @@ class AccessLog < LogfileInterval::ParsedLine::Base
   add_column :name => 'code',         :pos => 4, :aggregator => :count
   add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
 
+  skip                                :pos => 3, :regex => /firefox/
+
   def time
     DateTime.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z').to_time
   end
@@ -97,6 +99,8 @@ class AccessLog < LogfileInterval::ParsedLine::Base
   add_column :name => 'length',       :pos => 5, :aggregator => :average,                      :conversion => :integer
   add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => 'ip', :conversion => :integer
 
+  skip                                :pos => 3, :regex => /firefox/
+
   def time
     Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
   end
@@ -105,6 +109,7 @@ end
 #### The parser must define:
 * A regex that extracts the fields out of each line.
 * A set of columns that will to be parsed and aggregated in time intervals.
+* 0 or more column that will be skipped if the column value matches the specified regex
 * A 'time' method that converts the mandatory timestamp field of a line into a Time object.
 
 #### Attributes of a column:

data/bin/aggregate_access_log.rb

@@ -18,9 +18,9 @@ class AccessLogParsedLine < LogfileInterval::ParsedLine::Base
 
   add_column :name => 'ip',           :pos => 1, :aggregator => :count
   add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
-  add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
-  add_column :name => 'length',       :pos => 5, :aggregator => :average,                      :conversion => :integer
-  add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => 'ip', :conversion => :integer
+  add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => :ip
+  add_column :name => 'length',       :pos => 5, :aggregator => :average,                     :conversion => :integer
+  add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => :ip, :conversion => :integer
   add_column :name => 'referer',      :pos => 6, :aggregator => :count
   add_column :name => 'referer_by_ip', :pos => 6, :aggregator => :count,    :group_by => :ip
 
@@ -40,4 +40,3 @@ builder.each_interval do |interval|
   pp interval[:referer_by_ip]
   STDIN.gets
 end
-

data/lib/logfile_interval/aggregator_set.rb

@@ -12,6 +12,8 @@ module LogfileInterval
     def add(record)
       @parser_columns.each do |name, options|
         next unless @aggregators[name]
+        next unless options[:noskip] || !record.skip_with_exceptions?
+
         group_by_value = record[options[:group_by]] if options[:group_by]
         @aggregators[name].add(record[name], group_by_value)
       end

data/lib/logfile_interval/interval_builder.rb

@@ -20,9 +20,8 @@ module LogfileInterval
 
       case order
       when :asc  then self.extend Ascending
-      when :desc then self.extend Descending
-      when :empty then nil
-      else raise ArgumentError, "Can't determine parsed_lines_enum sort order"
+      else
+        self.extend Descending
       end
     end
 

data/lib/logfile_interval/logfile.rb

@@ -43,7 +43,7 @@ module LogfileInterval
       return enum_for(:each_parsed_line) unless block_given?
       each_line do |line|
         record = parser.create_record(line)
-        yield record if record
+        yield record if record && !record.skip?
       end
     end
     alias_method :each, :each_parsed_line

data/lib/logfile_interval/parsed_line/base.rb

@@ -10,12 +10,22 @@ module LogfileInterval
       def initialize(line)
         @data = self.class.parse(line)
         @valid = @data ? true : false
+        @skip = @data ? @data[:skip] : false
+        @skip_with_exceptions = @data ? @data[:skip_with_exceptions] : false
       end
 
       def valid?
         @valid
       end
 
+      def skip?
+        @skip
+      end
+
+      def skip_with_exceptions?
+        @skip_with_exceptions
+      end
+
       def time
         raise NotImplemented
       end
@@ -26,6 +36,3 @@ module LogfileInterval
     end
   end
 end
-
-
-

data/lib/logfile_interval/parsed_line/parser.rb

@@ -3,95 +3,132 @@ module LogfileInterval
     class ConfigurationError  < StandardError; end
 
     module Parser
-        attr_reader :regex
-        def columns
-          @columns ||= {}
-        end
-
-        def set_regex(regex)
-          @regex = regex
-        end
+      attr_reader :regex
 
-        def add_column(options)
-          validate_column_options(options)
-          options = sanitize_column_options(options)
+      def columns
+        @columns ||= {}
+      end
 
-          name = options[:name]
-          columns[name] = options
+      def skip_columns
+        @skip_columns ||= []
+      end
 
-          define_method(name) do
-            @data[name]
-          end
-        end
+      def skip_columns_with_exceptions
+        @skip_columns_with_exceptions ||= []
+      end
 
-        def parse(line)
-          raise ConfigurationError, 'There must be at least 1 configured column' unless columns.any?
-          raise ConfigurationError, 'A regex must be set' unless regex
+      def set_regex(regex)
+        @regex = regex
+      end
 
-          match_data = regex.match(line)
-          return nil unless match_data
+      def add_column(options)
+        validate_column_options(options)
+        options = sanitize_column_options(options)
 
-          data = {}
-          columns.each do |name, options|
-            val = match_data[options[:pos]]
-            data[name] = convert(val, options[:conversion])
-          end
-          data
-        end
+        name = options[:name]
+        columns[name] = options
 
-        def create_record(line)
-          record = new(line)
-          return record if record.valid?
-          return nil
+        define_method(name) do
+          @data[name]
         end
+      end
 
-        def set_column_custom_options(column_name, options)
-          raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
-          columns[column_name][:custom_options] = options
+      def skip(options)
+        unless options[:pos] && options[:regex]
+          raise ConfigurationError, "skip option must include pos and regex"
         end
 
+        skip_columns << { pos: options[:pos], regex: options[:regex] }
+      end
 
-        def each(&block)
-          columns.each(&block)
+      def skip_with_exceptions(options)
+        unless options[:pos] && options[:regex]
+          raise ConfigurationError, "skip option must include pos and regex"
         end
 
-        private
+        skip_columns_with_exceptions << { pos: options[:pos], regex: options[:regex] }
+      end
 
-        def validate_column_options(options)
-          validate_option(options, :name)
-          validate_option(options, :pos)
-          validate_option(options, :aggregator)
-          unless Aggregator::Base.exist?(options[:aggregator]) || options[:aggregator] == :timestamp
-            raise ConfigurationError, "aggregator must be one of #{Aggregator::Base.all.join(', ')}"
-          end
-        end
+      def parse(line)
+        raise ConfigurationError, 'There must be at least 1 configured column' unless columns.any?
+        raise ConfigurationError, 'A regex must be set' unless regex
+
+        match_data = regex.match(line)
+        return nil unless match_data
 
-        def validate_option(options, key, errmsg = nil)
-          raise ConfigurationError, errmsg || "#{key} is a mandatory column option" unless options.has_key?(key)
+        data = { skip: false }
+        columns.each do |name, options|
+          val = match_data[options[:pos]]
+          data[name] = convert(val, options[:conversion])
         end
 
-        def sanitize_column_options(options)
-          options[:name]       = options[:name].to_sym
-          if options.has_key?(:group_by)
-            if options[:group_by].to_sym != options[:name]
-              options[:group_by] = options[:group_by].to_sym
-            else
-              options.delete(:group_by)
-            end
+        skip_columns.each do |options|
+          val = match_data[options[:pos]]
+          if val =~ options[:regex]
+            data[:skip] = true
+            break
           end
-          options[:conversion] = options.fetch(:conversion, :string)
-          options[:aggregator_class] = Aggregator::Base.klass(options[:aggregator])
-          options.delete(:aggregator)
-          options
         end
 
-        def convert(val, conversion)
-          case conversion
-          when :integer then val.to_i
-          when :float   then val.to_f
-          else val
+        data
+      end
+
+      def create_record(line)
+        record = new(line)
+        return record if record.valid?
+        return nil
+      end
+
+      def set_column_custom_options(column_name, options)
+        raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
+        columns[column_name][:custom_options] = options
+      end
+
+
+      def each(&block)
+        columns.each(&block)
+      end
+
+      private
+
+      def validate_column_options(options)
+        validate_option(options, :name)
+        validate_option(options, :pos)
+        validate_option(options, :aggregator)
+        if options[:name].to_s == 'skip'
+          raise ConfigurationError, "'skip' is a reserved column name"
+        end
+        unless Aggregator::Base.exist?(options[:aggregator]) || options[:aggregator] == :timestamp
+          raise ConfigurationError, "aggregator must be one of #{Aggregator::Base.all.join(', ')}"
+        end
+      end
+
+      def validate_option(options, key, errmsg = nil)
+        raise ConfigurationError, errmsg || "#{key} is a mandatory column option" unless options.has_key?(key)
+      end
+
+      def sanitize_column_options(options)
+        options[:name]       = options[:name].to_sym
+        if options.has_key?(:group_by)
+          if options[:group_by].to_sym != options[:name]
+            options[:group_by] = options[:group_by].to_sym
+          else
+            options.delete(:group_by)
           end
         end
+        options[:conversion] = options.fetch(:conversion, :string)
+        options[:aggregator_class] = Aggregator::Base.klass(options[:aggregator])
+        options.delete(:aggregator)
+        options
+      end
+
+      def convert(val, conversion)
+        case conversion
+        when :integer then val.to_i
+        when :float   then val.to_f
+        else val
+        end
+      end
     end
   end
 end

data/lib/logfile_interval/version.rb

@@ -1,3 +1,3 @@
 module LogfileInterval
-  VERSION = "2.1.5"
+  VERSION = "2.2.1"
 end

data/spec/lib/logfile_spec.rb

@@ -78,6 +78,15 @@ module LogfileInterval
         records.last.length_by_ip.should == 185
       end
 
+      it 'skips lines matching skip options' do
+        records = []
+        @alf.each_parsed_line do |record|
+          records << record
+        end
+
+        records.size.should == 6
+      end
+
       context 'without a block' do
         it 'should return an enumerator' do
           e = @alf.each_parsed_line

data/spec/support/lib/access_log.rb

@@ -15,6 +15,8 @@ module LogfileInterval
       add_column :name => 'length',       :pos => 5, :aggregator => :average,   :conversion => :integer
       add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :conversion => :integer, :group_by => 'ip'
 
+      skip :pos => 7, :regex => /Spinn3r/
+
       def time
         Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logfile_interval
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.2.1
 platform: ruby
 authors:
 - Philippe Le Rohellec
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2018-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,7 +164,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Aggregate logfile data into intervals