ascii-data-tools 0.9

data/features/step_definitions/ascii-data-cat_steps.rb

@@ -0,0 +1,48 @@
+require 'stringio'
+
+Before do |scenario|
+  @output_stream = StringIO.new
+  @input_stream = StringIO.new
+  @user_feedback_stream = StringIO.new
+  
+  @command_line = []
+  
+  AsciiDataTools.record_types.clear
+  load 'ascii-data-tools/discover.rb'
+  @record_types = AsciiDataTools.record_types
+end
+
+Given /^a record stream containing$/ do |string|
+  @input_stream.string = string
+end
+
+Given /^file "([^\"]*)" containing$/ do |filename, string|
+  @record_source_filename = filename
+  @input_stream.string = string
+end
+
+When /^ascii-data-cat is invoked$/ do
+  AsciiDataTools::Controller::CatController.new(
+    :input_sources => [AsciiDataTools::InputSource.new(@record_source_filename, @input_stream)],
+    :output_stream => @output_stream,
+    :record_types  => @record_types
+  ).run
+end
+
+When /^([^\"]*) is invoked on a file "([^\"]*)" containing$/ do |executable, filename, string|
+  Given "file \"#{filename}\" containing", string
+  When "#{executable} is invoked"
+end
+
+When /^([^\"]*) is invoked on a record stream containing$/ do |executable, string|
+  Given "a record stream containing", string
+  When "#{executable} is invoked"
+end
+
+Then /^the following is printed out:$/ do |string|
+  @output_stream.string.should == string
+end
+
+Then /^the user receives the following feedback:$/ do |string|
+  @user_feedback_stream.string.should == string
+end

data/features/step_definitions/ascii-data-edit_steps.rb

@@ -0,0 +1,38 @@
+When /^ascii-data-edit is invoked$/ do
+  AsciiDataTools::Controller::EditController.new(
+    :input_sources        => [AsciiDataTools::InputSource.new(@record_source_filename, @input_stream)],
+    :output_stream        => @output_stream,
+    :record_types         => @record_types,
+    :user_feedback_stream => @user_feedback_stream,
+    :editor               => lambda do |filenames|
+                               edited_file = File.new(filenames.first)
+                               edited_file.extend(AsciiDataTools::ExternalPrograms)
+                               @text_prior_to_edit = edited_file.read
+                               File.open(edited_file.path, 'w') {|f| f << @text_after_edit}
+                               edited_file.modify_file_mtime_to(Time.now + 1)
+                             end
+  ).run
+end
+
+When /^the output is successfully ascii\-edited to the following:$/ do |string|
+  @text_after_edit = string
+  When "ascii-data-edit is invoked"
+end
+
+When /^the output is ascii\-edited without alteration$/ do
+  AsciiDataTools::Controller::EditController.new(
+    :input_sources        => [AsciiDataTools::InputSource.new(@record_source_filename, @input_stream)],
+    :output_stream        => @output_stream,
+    :record_types         => @record_types,
+    :user_feedback_stream => @user_feedback_stream,
+    :editor               => lambda do |filenames| end
+  ).run
+end
+
+Then /^the editor shows:$/ do |string|
+  @text_prior_to_edit.should == string
+end
+
+Then /^the encoded record stream contains:$/ do |string|
+  @output_stream.string.should == string
+end

data/features/step_definitions/ascii-data-norm_steps.rb

@@ -0,0 +1,7 @@
+When /^ascii\-data\-norm is invoked$/ do
+  AsciiDataTools::Controller::NormalisationController.new(
+    :input_sources => [AsciiDataTools::InputSource.new(@record_source_filename, @input_stream)],
+    :output_stream => @output_stream,
+    :record_types  => @record_types
+  ).run
+end

data/features/step_definitions/ascii-data-qdiff_steps.rb

@@ -0,0 +1,43 @@
+Given /^streams containing$/ do |text|
+  @input_stream1 = StringIO.new
+  @input_stream2 = StringIO.new
+  
+  text.split("\n").each do |line|
+    left_line, right_line = line.split("||").map(&:strip)
+    @input_stream1 << left_line.gsub('\n', "\n") unless left_line == "-" * left_line.length and not left_line.empty?
+    @input_stream2 << right_line.gsub('\n', "\n") unless right_line == "-" * right_line.length and not right_line.empty?
+  end
+  @input_stream1.rewind
+  @input_stream2.rewind
+end
+
+When /^ascii\-data\-qdiff is invoked on files containing:$/ do |string|
+  Given "streams containing", string
+  When "ascii-data-qdiff is invoked"
+end
+
+When /^ascii\-data\-qdiff is invoked$/ do
+  @actual_output1 = @actual_output2 = nil
+  AsciiDataTools::Controller::QDiffController.new(
+    :input_sources        => [AsciiDataTools::InputSource.new(nil, @input_stream1),
+                              AsciiDataTools::InputSource.new(nil, @input_stream2)],
+    :editor               => lambda do |filenames|
+                               @actual_output1 = File.read(filenames.first)
+                               @actual_output2 = File.read(filenames.last)
+                             end,
+    :output_stream        => @output_stream,
+    :record_types         => @record_types,
+    :user_feedback_stream => @user_feedback_stream
+  ).run
+end
+
+Then /^the diffed result should be:$/ do |text|
+  expected_output1, expected_output2 = "", ""
+  text.split("\n").each do |line|
+    left_line, right_line = line.split("||").map(&:strip)
+    expected_output1 << left_line + "\n" unless left_line == "-" * left_line.length and not left_line.empty?
+    expected_output2 << right_line + "\n" unless right_line == "-" * right_line.length and not right_line.empty?
+  end
+  @actual_output1.should == expected_output1
+  @actual_output2.should == expected_output2
+end

data/features/step_definitions/encoding_decoding_steps.rb

@@ -0,0 +1,23 @@
+When /^I decode an encoded record "([^\"]*)" of type "([^\"]*)"$/ do |ascii_text, record_type_name|
+  ascii_text.gsub!('\\n', "\n")
+  @record = @record_types.find_by_name(record_type_name).decode(:ascii_string => ascii_text)
+end
+
+When /^I encode a record of type "([^\"]*)" and contents:$/ do |record_type_name, table|
+  record_type = @record_types.find_by_name(record_type_name)
+  values = table.hashes.collect {|hash| hash["field value"].gsub('\\n', "\n") }
+  @record = AsciiDataTools::Record::Record.new(record_type, values)
+end
+
+Then /^I should have a decoded record of type "([^\"]*)" and contents:$/ do |intended_record_type_name, table|
+  @record.type_name.should == intended_record_type_name
+  table.hashes.each do |hash|
+    expected_value = hash["field value"].gsub('\\n', "\n")
+    @record[hash["field name"]].should == expected_value
+  end
+end
+
+Then /^I should have a encoded record "([^\"]*)"$/ do |expected_encoded_text|
+  expected_encoded_text.gsub!('\\n', "\n")
+  @record.encode.should == expected_encoded_text
+end

data/features/step_definitions/plugins_steps.rb

@@ -0,0 +1,11 @@
+Given /^the following configuration:$/ do |code|
+  @record_types.instance_eval(code)
+end
+
+When /^the record type configuration is printed$/ do
+  @configuration_printout = AsciiDataTools::RecordTypesConfigurationPrinter.for_record_types(@record_types).summary
+end
+
+Then /^it should look like this:$/ do |expected_string|
+  @configuration_printout.should == expected_string
+end

data/features/step_definitions/record_recognition_steps.rb

@@ -0,0 +1,10 @@
+When /^record "([^\"]*)" coming from ([^\"]*) is analysed$/ do |ascii_text, record_source_name|
+  record_source_name = nil if record_source_name == "unspecified"
+  ascii_text.gsub!('\\n', "\n")
+  type_determiner = AsciiDataTools::RecordType::TypeDeterminer.new(@record_types)
+  @type = type_determiner.determine_type_for(:ascii_string => ascii_text, :filename => record_source_name)
+end
+
+Then /^its type should be recognised as "([^\"]*)"$/ do |type_name|
+  @type.name.should == type_name
+end

data/features/support/env.rb

@@ -0,0 +1,5 @@
+lib_path = File.expand_path("#{File.dirname(__FILE__)}/../../lib")
+$LOAD_PATH.unshift lib_path unless $LOAD_PATH.include?(lib_path)
+
+require 'ascii-data-tools'
+require 'ascii-data-tools/configuration_printer'

data/lib/ascii-data-tools.rb

@@ -0,0 +1,8 @@
+require 'ascii-data-tools/ruby_extensions'
+
+require 'ascii-data-tools/record'
+require 'ascii-data-tools/record_type'
+require 'ascii-data-tools/formatting'
+require 'ascii-data-tools/configuration'
+require 'ascii-data-tools/global_autodiscovery'
+require 'ascii-data-tools/controller'

data/lib/ascii-data-tools/configuration.rb

@@ -0,0 +1,169 @@
+require 'optparse'
+require 'zlib'
+require 'tempfile'
+
+module AsciiDataTools
+  class Configuration
+    attr_reader :input_sources, :output_stream, :errors, :record_types, :editor, :user_feedback_stream
+    
+    def initialize(arguments, overrides = {})
+      @arguments = arguments
+      @overrides = overrides
+      @errors = []
+
+      @opts = define_optionparser_configuration
+      remainder = parse(arguments)
+
+      @output_stream        = overrides[:output_stream] || STDOUT
+      @input_sources        = overrides[:input_sources] || make_input_streams(remainder, overrides)
+      @record_types         = overrides[:record_types]  || load_record_types
+      @editor               = overrides[:editor]
+      @user_feedback_stream = overrides[:user_feedback_stream] || STDOUT
+    end
+    
+    def valid?
+      @errors.empty?
+    end
+    
+    def error_info_with_usage
+      @errors.each {|error| puts error}
+      puts
+      puts @opts
+      exit 1
+    end
+    
+    protected
+    def make_input_streams(remainder, overrides)
+      begin
+        return InputSourceFactory.new(overrides).input_sources_from(remainder)
+      rescue Exception => e
+        @errors << e.message
+        return nil
+      end       
+    end
+    
+    def define_optionparser_configuration
+      OptionParser.new do |opts|
+        opts.banner = [
+          "Usage: #{File.basename($0)} [options] <input source>",
+          "An input source can be either a flat file or a gzipped flat file.",
+          @overrides[:input_pipe_accepted] ? "For this command, - (STDIN) is also allowed." : nil,
+          "\n"].compact.join("\n")
+        
+        opts.separator ""
+        opts.separator "Other options:"
+        
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end
+    end
+    
+    def parse(arguments)
+      begin
+        return @opts.parse(arguments)
+      rescue SystemExit => e
+        exit e.status
+      rescue Exception => e
+        @errors << e.message
+        return []
+      end
+    end
+    
+    def load_record_types
+      AsciiDataTools.autodiscover
+      AsciiDataTools.record_types
+    end
+  end
+  
+  class InputSourceFactory
+    def initialize(properties = {})
+      @expected_argument_number = properties[:expected_argument_number] || 1
+      @input_pipe_accepted = properties[:input_pipe_accepted].nil? ? true : properties[:input_pipe_accepted]
+    end
+    
+    def input_sources_from(input_arguments)
+      validate_number_of input_arguments
+      return input_arguments.collect {|arg| make_input_source_from(arg)}
+    end
+    
+    protected
+    def validate_number_of(input_arguments)
+      raise "No input specified." if input_arguments.empty?
+      error_message = "#{input_arguments.size} input sources detected: #{input_arguments.inspect}. " +
+                      "This command accepts #{@expected_argument_number} input source(s)."
+      raise error_message if input_arguments.length != @expected_argument_number
+    end
+    
+    def make_input_source_from(input_argument)
+      if input_argument == "-"
+        if @input_pipe_accepted
+          return InputSource.new(nil, STDIN)
+        else
+          raise "STDIN not accepted for this command."
+        end
+      end
+      
+      path_to_file = input_argument
+      raise "File #{path_to_file} does not exist!" unless File.exists?(path_to_file)
+      return InputSource.new(path_to_file, Zlib::GzipReader.open(path_to_file)) if path_to_file =~ /[.]gz$/
+      return InputSource.new(path_to_file, File.open(path_to_file))
+    end
+  end
+  
+  class InputSource < Struct.new(:filename, :stream)
+    def read
+      stream.readline
+    end
+    
+    def has_records?
+      not stream.eof?
+    end
+  end
+  
+  class Editor
+    def initialize(&edit_command)
+      @tempfiles = {}
+      @preedit_mtimes = {}
+      @postedit_mtimes = {}
+      @edit_command = edit_command
+    end
+    
+    def [](n)
+      @tempfiles[n] ||= Tempfile.new("ascii_tools")
+    end
+    
+    def edit
+      close_all_tempfiles
+      save_preedit_mtimes
+      edit_files
+      save_postedit_mtimes
+    end
+    
+    def changed?(n)
+      not @preedit_mtimes[n] == @postedit_mtimes[n]
+    end
+    
+    protected
+    def close_all_tempfiles
+      @tempfiles.values.each {|f| f.close }
+    end
+    
+    def save_preedit_mtimes
+      @tempfiles.each {|n, f| @preedit_mtimes[n] = File.mtime(f.path)}
+    end
+
+    def save_postedit_mtimes
+      @tempfiles.each {|n, f| @postedit_mtimes[n] = File.mtime(f.path)}
+    end
+    
+    def edit_files
+      @edit_command[sorted_filenames]
+    end
+    
+    def sorted_filenames
+      @tempfiles.sort.collect {|number, tempfile| tempfile.path}
+    end
+  end
+end

data/lib/ascii-data-tools/configuration_printer.rb

@@ -0,0 +1,38 @@
+require 'terminal-table/import'
+
+module AsciiDataTools
+  class RecordTypesConfigurationPrinter
+    def initialize(presenter)
+      @presenter = presenter
+    end
+    
+    def summary
+      table do |t|
+        t.headings = @presenter.headings
+        @presenter.record_type_summaries.each {|summary| t << summary}
+      end.to_s
+    end
+        
+    class << self
+      def for_record_types(record_types)
+        new(RecordTypesConfigurationPresenter.new(record_types))
+      end
+    end
+  end
+  
+  class RecordTypesConfigurationPresenter
+    def initialize(record_types)
+      @record_types = record_types
+    end
+    
+    def headings
+      ["type name", "total length", "constraints", "normalised fields"]
+    end
+    
+    def record_type_summaries
+      @record_types.sort_by {|record_type| record_type.total_length_of_fields}.inject([]) do |summaries, record_type|
+        summaries << [record_type.name, record_type.total_length_of_fields, record_type.constraints_description, record_type.names_of_normalised_fields]
+      end
+    end
+  end  
+end

data/lib/ascii-data-tools/controller.rb

@@ -0,0 +1,123 @@
+require 'ascii-data-tools/filter'
+require 'ascii-data-tools/filter/diffing'
+require 'ascii-data-tools/external_programs'
+
+module AsciiDataTools
+  module Controller
+    class AbstractController
+      def initialize(configuration_or_command_line_arguments)
+        case configuration_or_command_line_arguments
+        when Hash then
+          @configuration = Configuration.new([], defaults.merge(configuration_or_command_line_arguments))
+        when Array then
+          @configuration = Configuration.new(configuration_or_command_line_arguments, defaults)
+        when Configuration then
+          @configuration = configuration_or_command_line_arguments
+        end
+        @configuration.error_info_with_usage unless @configuration.valid?
+      end
+
+      def type_determiner
+        @type_determiner ||= RecordType::TypeDeterminer.new(@configuration.record_types)
+      end
+      
+      def run
+        raise "should be implemented!"
+      end
+      
+      protected
+      def input_source
+        @configuration.input_sources.first
+      end
+      
+      def defaults
+        {:expected_argument_number => 1, :input_pipe_accepted => true}
+      end
+    end
+    
+    class CatController < AbstractController
+      def run
+        formatting_filter = Filter::FormattingFilter.new(input_source.filename, type_determiner)
+        formatting_filter << input_source
+        formatting_filter.write(@configuration.output_stream)
+      end
+    end
+    
+    class EditController < AbstractController
+      include ExternalPrograms
+      include Filter
+
+      def run
+        editor = Editor.new(&@configuration.editor)
+        formatting_filter = FormattingFilter.new(input_source.filename, type_determiner)
+        formatting_filter << input_source
+        
+        formatting_filter.write(editor[0])
+        editor.edit
+        
+        if not editor.changed?(0)        
+          @configuration.user_feedback_stream.puts "The file is unmodified."
+        else
+          encoding_filter = Filter::Filter.new {|record| record.encode }
+          parsing_filter = ParsingFilter.new(@configuration.record_types)
+          encoding_filter << (parsing_filter << InputSource.new(nil, editor[0].open))
+          encoding_filter.write(output_stream)
+        end
+      end
+      
+      protected
+      def output_stream
+        @configuration.output_stream == STDOUT ? File.open(input_source.filename, 'w') : @configuration.output_stream
+      end
+      
+      def defaults
+        {:expected_argument_number => 1,
+         :input_pipe_accepted => false,
+         :editor => lambda {|filenames| edit_differences(filenames)} }
+      end
+    end
+    
+    class NormalisationController < AbstractController
+      def run
+        normalising_filter = Filter::NormalisingFilter.new(input_source.filename, type_determiner)
+        normalising_filter << input_source
+        normalising_filter.write(@configuration.output_stream)
+      end
+    end
+    
+    class QDiffController < AbstractController
+      include ExternalPrograms
+      include Filter
+      include Filter::Diffing
+      
+      def run
+        editor = Editor.new(&@configuration.editor)
+
+        normaliser1    = NormalisingFilter.new( @configuration.input_sources[0].filename, type_determiner)
+        normaliser2    = NormalisingFilter.new( @configuration.input_sources[1].filename, type_determiner)
+        sorter1        = SortingFilter.new
+        sorter2        = SortingFilter.new
+        diff_executer  = DiffExecutingFilter.new
+        diff_parser    = DiffParsingFilter.new
+        diff_formatter = DiffFormattingFilter.new(type_determiner)
+        
+        diff_formatter << (diff_parser << (diff_executer << [sorter1 << (normaliser1 << @configuration.input_sources[0]),
+                                                             sorter2 << (normaliser2 << @configuration.input_sources[1])]))
+        
+        begin
+          diff_formatter.write(editor[0], editor[1])
+          editor.edit
+        rescue StreamsEqualException => e
+          @configuration.user_feedback_stream.puts "The files are identical."
+        end        
+      end
+      
+      protected
+      def defaults
+        {:expected_argument_number => 2,
+         :input_pipe_accepted => false,
+         :editor => lambda {|filenames| edit_differences(filenames)} }
+      end
+    end
+  end
+end