syc-spector 0.0.1

data/lib/inspector/options.rb

@@ -0,0 +1,212 @@
+require 'optparse'
+require 'io/wait'
+
+require_relative 'pattern.rb'
+
+# Module Inspector contains the functions related to the command line parsing.
+# Provides the parsed user input in the options hash.
+module Inspector
+
+  # Parses the provided command line flags, switches and arguments
+  class Options
+    # Options hold all the parameters used by the application
+    attr_reader :options
+
+    # Initializes Options and parses the provided user input from the command
+    # line
+    def initialize(argv)
+      parse(argv)
+    end
+
+    # After running the application output files are saved in the history.
+    # --get_file_from_history-- retrieves the valid or invalid file from the
+    # last application invokation. Where type is eather +valid+ or +invalid+.
+    #
+    # :call-seq:
+    #   get_file_from_history(type)
+    def get_file_from_history(type)
+      pattern = Regexp.new('_'+type.to_s)
+      File.open(".sycspector.data", 'r') do |file|
+        while name = file.gets.chop
+          return name unless name.scan(pattern).empty?
+        end
+      end
+    end
+
+    # Save the results of the last run of the application to +.sycspector.data+.
+    # It contains the valid and invalid file as well as the pattern.
+    def save_result_files(opts)
+      File.open(".sycspector.data", 'w') do |file|
+        file.puts opts[:valid_file]
+        file.puts opts[:invalid_file]
+        file.puts opts[:pattern].to_s
+        file.puts opts[:scan_pattern].to_s
+      end
+    end
+
+  private
+
+    # Creates the output files valid and invalid files based on the input file's
+    # name. If the filename already exists it is copied with a new timestamp.
+    # 
+    # :call-seq:
+    #   create_output_files(filename)
+    def create_output_files(filename)
+      timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
+      if filename =~ /\d{8}-\d{6}/
+        `cp #{filename} #{filename.sub(/\d{8}-\d{6}/, timestamp)}`
+        filename.slice!(/\d{8}-\d{6}_(valid|invalid)_/)
+      end
+      files = {valid_file: "#{timestamp}_valid_#{filename}",
+               invalid_file: "#{timestamp}_invalid_#{filename}"}
+    end
+ 
+    # Parses the provided arguments and stores the value in @options
+    #
+    # :call-seq:
+    #   parse(argv)
+    def parse(argv)
+      @options = {}
+      OptionParser.new do |opts|
+        opts.banner = "Usage: sycspector input_file [ options ]"
+
+        # create a switch
+
+        @options[:individualize] = false
+        opts.on("-i", "--individualize",
+                "Remove duplicate values") do
+            @options[:individualize] = true
+        end
+
+        @options[:sort] = false
+        opts.on("-s", "--sort",
+                "Sort values") do
+            @options[:sort] = true
+        end
+
+        @options[:fix] = false
+        opts.on("-f", "--fix", 
+                "prompt invalid values for fixing") do
+            @options[:fix] = true
+        end
+
+        @options[:mode] = 'w'
+        opts.on("-a", "--append",
+                "Append values to output file OUTFILE") do
+            @options[:mode] = 'a'
+        end
+        
+        # Create a flag
+
+        @options[:pattern] = /\A.*\Z/
+        @options[:scan_pattern] = /\A.*\Z/
+
+        opts.on("-p", "--pattern PATTERN",
+                "Values that match the pattern are",
+                "considered as valid values.",
+                "Default matches all.",
+                "'sycspector' -p email matches emails") do |pattern|
+          if pattern == 'email'
+            @options[:pattern] = EMAIL_PATTERN
+            @options[:scan_pattern] = ANY_EMAIL_PATTERN
+          else
+            @options[:pattern] = Regexp.new(pattern)
+            scan_pattern = pattern.match(FULL_LINE).to_s
+            puts "'#{scan_pattern}' '#{pattern}'"
+            pattern = scan_pattern unless scan_pattern.empty?
+            @options[:scan_pattern] = Regexp.new(pattern)
+          end
+        end
+        
+        @options[:delimiter] = ";"
+        opts.on("-d", "--delimiter DELIMITER", String, 
+                "Delimiter between values.",
+                "Default delimiter is ';'") do |delimiter|
+            @options[:delimiter] = delimiter || ";"
+        end
+
+        opts.on("-o", "--output OUTFILE", String,
+                "File name as basis for creation of valid",
+                "and invalid file name.",
+                "default '<timestamp>_valid_values,",
+                "<timestamp>_invalid_values'",
+                "where <timestamp> = 'YYmmDD-HHMMSS'") do |outfile|
+
+            files = create_output_files outfile
+            @options[:valid_file] = files[:valid_file]
+            @options[:invalid_file] = files[:invalid_file]
+        end
+
+        opts.on("--show [valid|invalid]", [:valid, :invalid],
+                "Show the last valid or invalid file",
+                "Default is valid") do |show|
+          @options[:show] = show || :valid
+         
+        end
+        
+        opts.on("-h", "--help", "Show this message") do
+            puts opts
+            exit(0)
+        end
+
+        begin
+            argv << "-h" if argv.empty?
+            opts.parse!(argv)
+            
+        rescue OptionParser::ParseError => e
+            STDERR.puts e.message, "\n", opts
+            exit(-1)
+        end
+        
+        if @options[:fix] and argv.empty?
+
+          files = {}
+          if File.exist?(".sycspector.data")
+            File.open(".sycspector.data", 'r') do |file|
+              files = create_output_files file.gets.chomp            
+              argv << file.gets.chomp
+              @options[:pattern] = Regexp.new(file.gets.chomp)
+              @options[:scan_pattern] = Regexp.new(file.gets.chomp)
+            end
+          else
+            STDERR.puts "--> no sycspector history.\n" +
+                        "    You first have to run sycspector FILENAME"
+            exit(-1)
+          end
+
+          unless files.empty?
+            @options[:valid_file] = files[:valid_file]
+            @options[:invalid_file] = files[:invalid_file]
+          end
+          
+        end
+        
+        if @options[:show] and argv.empty?
+          unless File.exist?(".sycspector.data")
+            STDERR.puts "--> no sycspector history." +
+                        "    You first have to run 'sycspector FILENAME'"
+            exit(-1)
+          end
+        else
+          @options[:infile] = argv.shift
+          
+          if @options[:infile].nil?
+            STDERR.puts "--> missing input file"
+            exit(-1)
+          end
+
+          unless File.exist?(@options[:infile])
+            STDERR.puts "--> infile '#{@options[:infile]}' does not exist"
+            exit(-1)
+          end
+          
+          if @options[:valid_file].nil? or @options[:invalid_file].nil?
+            files = create_output_files "values"
+            @options[:valid_file] = files[:valid_file]
+            @options[:invalid_file] = files[:invalid_file]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inspector/pattern.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# Pattern that relizes emails. Matches whole lines only
+EMAIL_PATTERN = /\A[\w!#\$%&'*+\/=?`{|}~^-]+(?:\.[\w!#\$%&'*+\/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}\Z/
+
+# Pattern that relizes an email within a line.
+ANY_EMAIL_PATTERN = /[\w!#\$%&'*+\/=?`{|}~^-]+(?:\.[\w!#\$%&'*+\/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}/
+
+# Pattern that relizes strings between \A and \Z that is beginning of line and
+# end of line patterns.
+FULL_LINE = /(?<=\\A).*(?=\\Z)/

data/lib/inspector/runner.rb

@@ -0,0 +1,31 @@
+require_relative 'options'
+require_relative 'separator'
+
+# Module Inspector contains functions related to running the application
+module Inspector
+  # Processes based on the provided command line arguments the scan of the
+  # file or shows the last processed files.
+  class Runner
+
+    # Initializes runner and invokes Options with the command line arguments
+    # provided
+    def initialize(argv)
+      @options = Options.new(argv)
+    end
+
+    # Runs the Separator or shows the requested valid or invalid file. To show
+    # the file 'less' is used.
+    def run
+      opts = @options.options
+      if opts[:infile]
+        separator = Separator.new
+        separator.process(opts)
+        separator.print_statistics(opts)
+        @options.save_result_files opts
+      end
+      if show = opts[:show]
+        system "less #{@options.get_file_from_history show}"
+      end
+    end
+  end
+end 

data/lib/inspector/separator.rb

@@ -0,0 +1,182 @@
+require_relative 'console'
+
+# This module encapsulates functionality related to scanning values for
+# patterns. If a line contains the pattern that line is added to valid files.
+# If the pattern is not found in the line the line is added to invalid files.
+# Lines can if requested to be fixed. The line is presented to the user and the
+# user can eather fix the value by typing the corrected value. Other
+# possibilities are to drop the line, scan the line or just considering the
+# line as valid so it is added to valid values.
+module Inspector
+
+  # The Separator scans the input file for a provided pattern and prints the
+  # results of the scan to the console.
+  class Separator
+    # The prompt string is presented to the user when values are requested to
+    # be fixed. (v)alid will add the value to the valid values without testing
+    # against the pattern. (i)nvalid adds the value to the invalid values.
+    # (d)rop discards the value, (s)can scans the line to look for the pattern.
+    # (f)ix allows to type the corrected value.
+    PROMPT_STRING =  "-> (v)alid (i)nvalid (d)rop (s)can (f)ix: " 
+
+    # Initializes the Separator and creating a Console object
+    def initialize
+      @console = Console.new
+    end
+
+    # Prints the results of the invokation of Separator.process
+    #
+    # :call-seq:
+    #   print_statistics(opts)
+    def print_statistics(opts)
+      puts "-> statistics"
+      puts "   ----------"
+      printf("%7s:   %4d\n%7s:   %4d\n%7s: %4d\n%7s:    %4d\n%7s:  %4d\n",
+             "   total", opts[:valid_counter] + 
+                         opts[:invalid_counter] +
+                         opts[:skip_counter] +
+                         opts[:double_counter],
+             "   valid", opts[:valid_counter],
+             "   invalid", opts[:invalid_counter],
+             "   drop", opts[:skip_counter],
+             "   double", opts[:double_counter])
+      puts
+      puts "-> pattern:      #{opts[:pattern].inspect}"
+      puts "-> scan pattern: #{opts[:scan_pattern].inspect}"
+      puts
+      puts "-> files operated on"
+      puts "   -----------------"
+      puts "   values read from:          #{opts[:infile]}"
+      puts "   valid values written to:   #{opts[:valid_file]}"
+      puts "   invalid values written to: #{opts[:invalid_file]}"
+      if opts[:note]
+        puts
+        puts "-> Note"
+        puts "   ----"
+        puts opts[:note]
+      end
+    end
+
+    # Prompts the PROMPT_STRING and tests the value against the pattern
+    #
+    # :call-seq:
+    #   fix(value, pattern) -> hash
+    #
+    # Return a hash with :value and :answer where :value contains the fixed
+    # value and the answer. To test whether the value is valid or invalid the
+    # :answer has to checked first.
+    def fix(value, pattern) 
+      choice = value
+      result = {}
+
+      while not pattern.match(choice)
+        puts "-> #{choice}?"
+        result[:answer] = @console.prompt PROMPT_STRING
+        case result[:answer]
+        when 'v'
+          result[:value] = choice
+          break
+        when 'i'
+          result[:value] = choice
+          break
+        when 'f'
+          print "-> fix: "
+          choice = gets.chomp
+          print "-> confirm "
+          redo
+        when 'd'
+          result[:value] = value
+          break
+        when 's'
+          result[:value] = value
+          break
+        end
+      end
+    
+      return result
+    end
+
+    # Processes the scan of the lines of the file and add the values eather to
+    # the valid or invalid values. The result of the scan will be added to the
+    # opts.
+    #
+    # :call-seq:
+    #   process(opts)
+    def process(opts) 
+      valid_file = File.open(opts[:valid_file], opts[:mode])
+      valid_values = []
+
+      invalid_file = File.open(opts[:invalid_file], 'w')
+      invalid_values = []
+      
+      skip_counter = 0
+
+      File.open(opts[:infile], 'r') do |file|
+        while line = file.gets
+          line.chomp.split(opts[:delimiter]).each do |value|
+              
+            match = value.match(opts[:pattern])
+
+            if match.nil? 
+              if opts[:fix]
+                result = fix value, opts[:pattern] 
+                case result[:answer]
+                when 'v'
+                  valid_values << result[:value]
+                when 'i'
+                  invalid_values << result[:value]
+                when 'd'
+                  skip_counter += 1
+                when 's'
+                  value.scan(opts[:scan_pattern]).each do |value|
+                    valid_values << value
+                  end
+                end
+              else
+                invalid_values << value
+              end
+            else
+              valid_values << match
+            end
+          end
+        end
+      end
+
+      valid_counter = valid_values.size
+
+      valid_values.uniq! {|value| value.downcase } if opts[:individualize]
+      valid_values.sort! if opts[:sort]
+
+      valid_values.each do |value|
+          valid_file.puts value
+      end
+
+      invalid_counter = invalid_values.size
+
+      invalid_values.uniq! {|value| value.downcase} if opts[:individualize]
+      invalid_values.sort! if opts[:sort]
+
+      invalid_values.each do |value|
+          invalid_file.puts value
+      end
+
+      valid_file.close
+      invalid_file.close
+      
+      double_counter = valid_counter - valid_values.size +
+                       invalid_counter - invalid_values.size
+
+      opts[:valid_counter] = valid_values.size 
+      opts[:invalid_counter] = invalid_values.size 
+      opts[:skip_counter] = skip_counter
+      opts[:double_counter] = double_counter
+      if (invalid_values.size > 0 and not opts[:fix])
+        opts[:note] = "   You can fix invalid values and append " + 
+                      "to valid with: $ sycspector -fa"
+      end
+
+    end
+
+  end
+
+end

data/sycspector.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |s|
+  s.name         = "syc-spector"
+  s.summary      = 
+    "Analyze a file and extract values matching a provided pattern.\n \
+     Allow to sort, remove double and manually fix values."
+  s.description  = File.read(File.join(File.dirname(__FILE__), 'README'))
+  s.requirements = ['No requirements']
+  s.version      = "0.0.1"
+  s.author       = "Pierre Sugar"
+  s.email        = "pierre@sugaryourcoffee.de"
+  s.homepage     = "http://syc.dyndns.org/drupal"
+  s.platform     = Gem::Platform::RUBY
+  s.required_ruby_version = '>=1.9'
+  s.files        = Dir['**/**']
+  s.executables  = ['sycspector']
+  s.test_files   = Dir['test/test*.rb']
+  s.has_rdoc     = true
+end

data/test/test_console.rb

@@ -0,0 +1,26 @@
+require 'test/unit'
+require 'shoulda'
+require_relative '../lib/inspector/console'
+
+# Tests the Console class
+class TestConsole < Test::Unit::TestCase
+
+  context "console" do
+    should "return value from prompt string" do
+      console = Inspector::Console.new
+      result = console.prompt "(v)alid, (i)nvalid, (d)rop, (s)can, (f)ix: "
+      choices = %w(v i d s f)
+      assert choices.index(result) >= 0
+    end
+
+    should "exit with code 0" do
+      console = Inspector::Console.new
+      begin
+        result = console.prompt "No choices are available. To end press Ctrl-c:"
+      rescue SystemExit => e
+        assert e.status == 0
+      end
+    end
+  end
+
+end

data/test/test_options.rb

@@ -0,0 +1,115 @@
+require 'test/unit'
+require 'shoulda'
+require_relative '../lib/inspector/options'
+require_relative '../lib/inspector/pattern'
+
+# Test of the Options class
+class TestOptions < Test::Unit::TestCase
+
+  context "specifying an inputfile that does not exist" do
+    should "exit with -1" do
+      begin
+        opts = Inspector::Options.new(["inputfile"])
+      rescue SystemExit => e
+        assert e.status == -1
+      end
+    end
+  end
+
+  context "specifying no input file" do
+    should "exit with -1" do
+      begin
+        opts = Inspector::Options.new([])
+      rescue SystemExit => e
+        assert e.status == 0
+      end
+    end
+  end
+
+  context "specifying an inputfile that exists" do
+    
+    # Intializes the input, valid, invalid and history file
+    def setup
+      puts "in startup"
+      File.open('existing', 'w') do |file|
+        file.puts "pierre@thesugars.de"
+        file.puts "amanda@thesugars.de und pierre@thesugars.de"
+      end
+      File.open('.sycspector.data', 'w') do |file|
+        file.puts "20130113-121212_valid_values"
+        file.puts "20130113-121212_invalid_values"
+        file.puts "(?-mix:\A[\w!#\$%&'*+\/=?`{|}~^-]+" +
+                  "(?:\.[\w!#\$%&'*+\/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\.)+" +
+                  "[a-zA-Z]{2,6}\Z)"
+        file.puts "(?-mix:[\w!#\$%&'*+\/=?`{|}~^-]+" +
+                  "(?:\.[\w!#\$%&'*+\/=?`{|}~^-]+)*@(?:[a-zA-Z0-9-]+\.)+" +
+                  "[a-zA-Z]{2,6})"
+      end
+      File.open('20130113-121212_valid_values', 'w') do |file|
+        file.puts "pierre@thesugars.de"
+        file.puts "amanda@thesugars.de"
+        file.puts "pierre@thesugars.de"
+      end
+      File.open('20130113-121212_invalid_values', 'w') do |file|
+        file.puts "amanda@thesugars und pierre@thesugars"
+      end 
+    end
+
+    # Cleans up the test directory by deleting the files created in setup
+    def teardown
+      puts "in shutdown"
+      `rm existing`
+      `rm 20130113-*`
+      `rm .sycspector.data`
+    end
+ 
+    should "return inputfile" do
+      opts = Inspector::Options.new(["existing"])
+      assert_equal "existing", opts.options[:infile]
+    end
+
+    should "return email pattern" do
+      opts = Inspector::Options.new(["-p", "email", "existing"])
+      assert_equal EMAIL_PATTERN, opts.options[:pattern]
+      assert_equal ANY_EMAIL_PATTERN, opts.options[:scan_pattern]
+    end
+    
+    should "return pattern and scan pattern" do
+      opts = Inspector::Options.new(["-p", "\\A\\w+.*\\d\\Z", "existing"])
+      assert_equal /\A\w+.*\d\Z/, opts.options[:pattern]
+      assert_equal /\w+.*\d/, opts.options[:scan_pattern]
+    end
+
+    should "return sort switch" do
+      opts = Inspector::Options.new(["-s", "existing"])
+      assert_equal true, opts.options[:sort]
+    end
+
+    should "return individualize switch" do
+      opts = Inspector::Options.new(["-i", "existing"])
+      assert_equal true, opts.options[:individualize]
+    end
+
+    should "return show :valid and infile nil" do
+      opts = Inspector::Options.new(["--show"])
+      assert_equal :valid, opts.options[:show]
+      assert_equal nil, opts.options[:infile]
+      opts = Inspector::Options.new(["--show", "valid"])
+      assert_equal nil, opts.options[:infile]
+    end
+
+    should "return show :invalid and infile nil" do
+      opts = Inspector::Options.new(["--show", "invalid"])
+      assert_equal :invalid, opts.options[:show]
+      assert_equal nil, opts.options[:infile]
+    end
+
+    should "return fix and invalid file as input file from last invokation" do
+      opts = Inspector::Options.new(["-f"])
+      assert_equal true, opts.options[:fix]
+      assert_equal "20130113-121212_invalid_values", opts.options[:infile]
+    end
+
+  end
+
+end