parsefiles 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0b06a9a7c1730baddc47203dd32b17432f42bc48
+  data.tar.gz: 21d776d817820a84c9480966a2f35e32f17a57cf
+SHA512:
+  metadata.gz: 7cf2a276067b81ac8663f5af2378dd958718d31a801993d101b14e7cd5956539aa8ad71f90e256aeb6d230c8d129d6b92a2f4e09fddefcc9de892dc25cf84fa7
+  data.tar.gz: 7d60c3a5640c6d2274dee16e3cbafee37b921edc7cafb0b8e2b96894b2202eeccf0a5fa16077aaa78f7f302f117fc505f85fae77faea86cbf9dde66557e480d2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+tags

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in parsefiles.gemspec
+gemspec
+
+
+gem 'guard-rspec'
+gem 'rb-fsevent', '~> 0.9.4'

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml|\.slim)$})          { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+
+  # Capybara features specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml|slim)$})     { |m| "spec/features/#{m[1]}_spec.rb" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Chris Simpson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Parsefiles
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'parsefiles'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install parsefiles
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/parsefiles

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'parsefiles'
+require 'cli'
+
+Parsefiles::CLI::Main.start
+

data/lib/cli.rb

@@ -0,0 +1,52 @@
+require 'date'
+require 'thor'
+require 'thor/runner'
+
+module Parsefiles
+  module CLI
+    class Main < Thor
+
+      desc 'version', 'Version of the Merge utility'
+      def version
+        require 'parsefiles/version'
+        say Parsefiles::VERSION
+      end
+
+      desc 'find', 'Find expression within specified text files'
+      long_desc <<-LONGDESC
+        Examples:
+        \x5 parsefiles find -d 2 -p simpson ./
+        \x5 => Search the current folder starting 2 days back for simpson.
+        \x5\x5 parsefiles find -s 08-01-2014 -e 08-03-2014 -p simpson ./
+        \x5 => Search between the 2 given dates for simpson
+        \x5\x5 parsefiles find -p simpson ./ /tmp/
+        \x5=> Search the current folder and tmp folder for simpson with today's date.
+      LONGDESC
+      method_option :days_back, aliases: '-d', type: 'string', default: '1', desc: 'number of days back to search'
+      method_option :start_date, aliases: '-s', type: 'string', default: Date.today.strftime('%m-%d-%Y'), desc: 'start date to begin file search from'
+      method_option :end_date, aliases: '-e', type: 'string', default: Date.today.strftime('%m-%d-%Y'), desc: 'end date to finish file search from'
+      method_option :pattern, aliases: '-p', type: 'string', default: ' ', desc: 'pattern to search for', :required => true
+
+      def find(*folders)
+        begin
+          folders = folders.split unless folders.class == Array
+          folders.each do |folder|
+            f = Parsefiles::Folders.new(folder,options)
+            files = f.get_files_in_range
+            files.each do |file|
+              puts "================================================"
+              puts "Reading : #{file}"
+              puts "================================================"
+              parse = Parsefiles::FileOps.new(file,options[:pattern])
+              parse.parse_file
+            end
+          end
+        rescue Exception => e
+          say e.message, :red
+        end
+      end
+
+    end
+  end
+
+end

data/lib/parsefiles/file_ops.rb

@@ -0,0 +1,41 @@
+require_relative 'parse'
+
+module Parsefiles
+  class FileOps
+
+    include Parse
+
+    attr_accessor :file,:pattern
+
+    def initialize(file,pattern)
+      @file = file
+      @pattern = pattern
+    end
+
+    def get_file_type
+      File.extname(@file).gsub!(/\./,'')
+    end
+
+    def parse_file
+      method_name = "parse_#{get_file_type}"
+      self.send method_name.to_sym
+    end
+
+    def match_line(line)
+      line = encode_string(line)
+      p line if line.match(@pattern)
+    end
+
+    def encode_string(input)
+      encode_it(input)
+    end
+
+    private
+
+    def encode_it(input)
+      input.force_encoding('UTF-8').encode('UTF-16', :invalid => :replace, :replace => '').encode('UTF-8')
+    end
+
+  end
+
+end

data/lib/parsefiles/folders.rb

@@ -0,0 +1,85 @@
+require 'date'
+require_relative 'utils'
+
+module Parsefiles
+  class Folders
+
+    include Utils
+
+    attr_accessor :folder, :files
+
+    VALID_TYPES = "{txt,log,gz}"
+
+    def initialize(folder,options={})
+      @folder = check_location(folder)
+      @files = get_files_with_date
+      @options = options
+    end
+
+    def check_location(folder)
+      if folder.match(/.*\.log$|txt$|gz$/i)
+        "./#{folder}"
+      else
+        folder = remove_trailing_slash(folder)
+        "#{folder}/*."+VALID_TYPES
+      end
+    end
+
+    def remove_trailing_slash(folder)
+      folder.gsub(/(.*)\/$/,'\1')
+    end
+
+    def get_files_in_range
+      start_date = get_start_date
+      end_date = get_end_date
+
+      result = []
+      @files.each_with_index do |file,index|
+        result << file[0] if Folders.date_in_range(file[1],start_date,end_date)
+      end
+      result
+    end
+
+    def get_start_date
+      if @options["days_back"].to_i > 1
+        days_back
+      else
+        start_date
+      end
+    end
+
+    def get_end_date
+      Folders.validate_date_format(@options["end_date"])
+      @options["end_date"] if @options["end_date"]
+    end
+
+    def calc_start_date(days_back)
+      date = Date.today - days_back
+      date.strftime('%m-%d-%Y')
+    end
+
+    private
+
+    def days_back
+      calc_start_date(@options[:days_back].to_i)
+    end
+
+    def start_date
+      Folders.validate_date_format(@options["start_date"])
+      @options["start_date"] if @options["start_date"]
+    end
+
+    # Returns an array with file and date in unix time format ['file.txt','40001245']
+    def get_files_with_date
+      file_list = Dir.glob(@folder)
+      result = []
+      file_list.reduce([]) do |file_stats,file|
+        file_stats = Array.new
+        file_stats << file
+        file_stats << Folders.unix_ts(file)
+        result << file_stats
+      end
+    end
+
+  end
+end

data/lib/parsefiles/parse.rb

@@ -0,0 +1,33 @@
+require 'zlib'
+
+module Parsefiles
+  module Parse
+
+
+    #Handles tar.gz and zip
+
+    def parse_gz
+      File.open(file, "r") do |f|
+        gz = Zlib::GzipReader.new(f)
+        gz.each_line do |line|
+          match_line(line.chomp)
+        end
+      end
+    end
+
+    def parse_zip
+      #TODO
+    end
+
+    def parse_log
+      File.open(@file, "r") do |open_file|
+        open_file.each do |line|
+          match_line(line.chomp)
+        end
+      end
+    end
+
+    alias_method :parse_txt, :parse_log
+
+  end
+end

data/lib/parsefiles/utils.rb

@@ -0,0 +1,32 @@
+module Parsefiles
+  module Utils
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+
+      def unix_ts(file)
+        File.mtime(file).to_i
+      end
+
+      def strDate_to_unix_timestamp(strDate)
+        Date.strptime(strDate, '%m-%d-%Y').to_time.to_i
+      end
+
+      def validate_date_format(strDate)
+        Date.strptime(strDate,'%m-%d-%Y')
+        rescue ArgumentError
+          raise "Invalid date format. Correct format : m-d-y."
+      end
+
+      def date_in_range(date,start_date,end_date)
+        start = strDate_to_unix_timestamp(start_date)
+        finish = strDate_to_unix_timestamp(end_date) + 86399
+        range = start..finish
+        range === date
+      end
+    end
+  end
+end

data/lib/parsefiles/version.rb

@@ -0,0 +1,3 @@
+module Parsefiles
+  VERSION = "0.0.8"
+end

data/lib/parsefiles.rb

@@ -0,0 +1,4 @@
+require 'parsefiles/file_ops.rb'
+require 'parsefiles/folders.rb'
+require 'parsefiles/parse.rb'
+require 'parsefiles/version.rb'

data/parsefiles.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'parsefiles/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "parsefiles"
+  spec.version       = Parsefiles::VERSION
+  spec.authors       = ["Chris Simpson"]
+  spec.email         = ["ctsimpson@gmail.com"]
+  spec.description   = "Utility for parsing log files"
+  spec.summary       = "Parse using regex with support for compressed files"
+  spec.homepage      = "http://www.ctsimpson.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.1"
+  spec.add_development_dependency "guard-rspec", "~> 4.2"
+  spec.add_development_dependency "rubyzip"
+  spec.add_dependency "thor"
+end

data/spec/fixtures/testfile.txt

@@ -0,0 +1,2 @@
+
+

data/spec/lib/cli_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'cli'
+
+module Parsefiles
+  module CLI
+
+    describe Main do
+
+    end
+
+  end
+end

data/spec/lib/parsefiles/file_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+module Parsefiles
+
+  describe FileOps do
+
+
+    let(:filename) { File.join('spec', 'fixtures', 'compressed.tar.gz')}
+    let(:file) {FileOps.new(filename,"test")}
+
+
+    context "when working with files" do
+
+      it "identifies the file type" do
+        expect(file.get_file_type).to eq "gz"
+      end
+
+      it "force UTF-8 encoding" do
+        string = "a\xFAb"
+        expect(file.encode_string(string)).to eq "ab"
+      end
+
+      it "calls parse method name based on file type" do
+        allow(file).to receive(:parse_gz).and_return ''
+        allow(file.parse_file).to receive(:parse_gz)
+      end
+    end
+
+    context 'when checking for matches' do
+
+      it "output the line when pattern is matched" do
+        line = "this do be a test"
+        expect(file.match_line(line)).to eq line
+      end
+    end
+
+    context 'when parsing zip files' do
+      let(:zip_file) { File.join('spec', 'fixtures', 'compressed.zip')}
+      let(:file) {FileOps.new(zip_file,"ctsimpso$")}
+
+      it "outputs matched content from a compressed zip file" do
+        skip "Zip files not yet implemented"
+      end
+    end
+  end
+end

data/spec/lib/parsefiles/folders_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+module Parsefiles
+
+  describe Folders do
+
+    context "when testing folder locations" do
+
+      let(:options) {{'start_date' => '06-27-2014', 'end_date' => '06-29-2014'}}
+
+      it "returns the folder location properly when a trailing slash is missing" do
+        test = Folders.new '/var/log', options
+        expect(test.folder).to eq '/var/log/*.{txt,log,gz}'
+      end
+
+      it "returns the folder location properly when their is a trailing slash" do
+        test = Folders.new '/var/log/', options
+        expect(test.folder).to eq '/var/log/*.{txt,log,gz}'
+      end
+
+      it "returns a single file name when a file name is passed in" do
+        test = Folders.new 'test.log', options
+        expect(test.folder).to eq './test.log'
+      end
+    end
+
+    context "when testing with start_date and end_date passed in" do
+
+      let(:options) {{'start_date' => '06-27-2014', 'end_date' => '06-29-2014'}}
+      let(:folder) {Folders.new './spec/fixtures/', options}
+
+      it "returns a list of files with date given a parent folder" do
+        output = [["./spec/fixtures/testfile.txt", 1406615852], ["./spec/fixtures/testfile2.txt", 1404000016], ["./spec/fixtures/compressed.tar.gz", 1407343892]]
+        expect(folder.files).to eq output
+      end
+
+      it "returns a list of files within a given date range" do
+        output = ['./spec/fixtures/testfile2.txt']
+        files = folder.get_files_in_range
+        expect(files).to eq output
+      end
+
+      it "calculates the start date based on days back attribute" do
+        allow(Date).to receive(:today).and_return(Date.parse('2014-07-20'))
+        expect(folder.calc_start_date 2).to eq '07-18-2014'
+      end
+
+      it "returns the start date from options if days back option is nil" do
+        expect(folder.get_start_date).to eq '06-27-2014'
+      end
+
+    end
+
+    context "when testing with days_back passed in" do
+
+      let(:options) {{"days_back"=>2}}
+      let(:folder) {Folders.new './spec/fixtures/', options}
+
+      it "returns the caculated start date if days_back is set" do
+        allow(folder).to receive(:days_back).and_return("07-14-2014")
+        expect(folder.get_start_date).to eq '07-14-2014'
+      end
+
+    end
+
+  end
+end

data/spec/lib/parsefiles/utils_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'parsefiles/utils'
+
+module Parsefiles
+
+  describe Utils do
+
+
+    before(:all) do
+      class Test
+        include Utils
+      end
+    end
+
+    it "converts a Date string to unix timestamp" do
+      expect(Test.strDate_to_unix_timestamp('01-01-2014')).to eq 1388563200
+    end
+
+    it "checks for a valid date format and raises an error if the date format is wrong" do
+      skip "Don't know why this test isn't working"
+      # expect(Test.validate_date_format('01-01d-2014')).to raise_error
+    end
+
+    it "checks for a valid date format and doesn't return nil for a valid date format" do
+      expect(Test.validate_date_format('01-01-2014')).to eq Date.parse('2014-01-01')
+    end
+
+    it "verifies that a date is within range of the start date and end date" do
+      skip "TODO"
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'rspec'
+require 'parsefiles'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/tasks.TODO

@@ -0,0 +1,7 @@
+ Planning:
+  ☐ Put together flow
+  ✘ Design user interface from cli @cancelled (12-06-13 10:35)
+  ☐ 
+
+Integration:
+ ☐ 

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: parsefiles
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- Chris Simpson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Utility for parsing log files
+email:
+- ctsimpson@gmail.com
+executables:
+- parsefiles
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/parsefiles
+- lib/.DS_Store
+- lib/cli.rb
+- lib/parsefiles.rb
+- lib/parsefiles/file_ops.rb
+- lib/parsefiles/folders.rb
+- lib/parsefiles/parse.rb
+- lib/parsefiles/utils.rb
+- lib/parsefiles/version.rb
+- parsefiles.gemspec
+- spec/.DS_Store
+- spec/fixtures/compressed.tar.gz
+- spec/fixtures/compressed.zip
+- spec/fixtures/testfile.txt
+- spec/fixtures/testfile2.txt
+- spec/lib/cli_spec.rb
+- spec/lib/parsefiles/file_spec.rb
+- spec/lib/parsefiles/folders_spec.rb
+- spec/lib/parsefiles/utils_spec.rb
+- spec/spec_helper.rb
+- tasks.TODO
+homepage: http://www.ctsimpson.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Parse using regex with support for compressed files
+test_files:
+- spec/.DS_Store
+- spec/fixtures/compressed.tar.gz
+- spec/fixtures/compressed.zip
+- spec/fixtures/testfile.txt
+- spec/fixtures/testfile2.txt
+- spec/lib/cli_spec.rb
+- spec/lib/parsefiles/file_spec.rb
+- spec/lib/parsefiles/folders_spec.rb
+- spec/lib/parsefiles/utils_spec.rb
+- spec/spec_helper.rb