ascii-data-tools 0.9

data/lib/ascii-data-tools/discover.rb

@@ -0,0 +1,19 @@
+AsciiDataTools.configure do
+  record_type("EXAMPLE01") do
+    field "RECORD_TYPE",      :length => 9, :constrained_to => "EXAMPLE01"
+    field "USAGE",            :length => 3
+    field "A_NUMBER",         :length => 16
+    field "B_NUMBER",         :length => 16
+    field "CHARGEABLE_UNITS", :length => 4
+    field "END_OF_RECORD",    :length => 1
+  end
+
+  record_type("EXAMPLE02") do
+    field "RECORD_TYPE",      :length => 9, :constrained_to => "EXAMPLE02"
+    field "APN",              :length => 12
+    field "TIMESTAMP",        :length => 14, :normalised => true
+    field "SESSION_DURATION", :length => 4
+    field "CHARGEABLE_UNITS", :length => 4
+    field "END_OF_RECORD",    :length => 1
+  end
+end

data/lib/ascii-data-tools/external_programs.rb

@@ -0,0 +1,23 @@
+module AsciiDataTools
+  module ExternalPrograms
+    def modify_file_mtime_to(new_mtime)
+      Kernel.system("touch -am -t #{new_mtime.strftime('%Y%m%d%H%M.%S')} #{self.path}")
+    end
+    
+    def diff(files)
+      IO.popen(diff_command_for(files))
+    end
+    
+    def diff_command_for(files)
+      "diff " + files.collect(&:path).join(' ')
+    end
+    
+    def sort(input_file, output_file)
+      Kernel.system("sort #{input_file.path} > #{output_file.path}")
+    end
+    
+    def edit_differences(filenames)
+      Kernel.system("vimdiff #{filenames.join(' ')}")
+    end
+  end
+end

data/lib/ascii-data-tools/filter.rb

@@ -0,0 +1,148 @@
+require 'tempfile'
+require 'ascii-data-tools/external_programs'
+
+module AsciiDataTools
+  module Filter
+    class Filter
+      def initialize(&block)
+        @filter_block = block
+      end
+      
+      def <<(upstream_filter)
+        @upstream = upstream_filter
+        self
+      end
+      
+      def read
+        filter(upstream.read)
+      end
+      
+      def write(output_stream)
+        output_stream << read while has_records?
+      end
+      
+      def has_records?
+        upstream.has_records?
+      end
+      
+      protected
+      def filter(record)
+        @filter_block[record]
+      end
+      
+      def upstream
+        @upstream
+      end
+    end
+    
+    class BufferingFilter < Filter
+      def initialize(&block)
+        @first_time = true
+        @filter_all_block = block
+      end
+      
+      protected
+      def filter_all(tempfile)
+        @filter_all_block[tempfile]
+      end
+      
+      def filter(record)
+        record
+      end
+      
+      def upstream
+        if @first_time
+          tempfile = buffer_as_tempfile(@upstream)
+          @upstream = InputSource.new(nil, filter_all(tempfile))
+          @first_time = false
+        end
+        @upstream
+      end
+      
+      def buffer_as_tempfile(stream)
+        tempfile = Tempfile.new("filter")
+        tempfile << stream.read while stream.has_records?
+        tempfile.open
+      end
+    end
+    
+    class FormattingFilter < Filter
+      def initialize(filename, type_determiner)
+        @formatter = Formatting::Formatter.new
+        @filename = filename
+        @type_determiner = type_determiner
+      end
+      
+      def filter(record)
+        encoded_record = {:ascii_string => record, :filename => @filename}
+        type = @type_determiner.determine_type_for(encoded_record)
+        decoded_record = type.decode(encoded_record)
+        @formatter.format(decoded_record)
+      end
+    end
+    
+    class NormalisingFilter < Filter
+      def initialize(filename, type_determiner)
+        @filename = filename
+        @type_determiner = type_determiner
+      end
+      
+      def filter(record)
+        type = @type_determiner.determine_type_for(:ascii_string => record, :filename => @filename)
+        type.normalise(record)
+      end
+    end
+    
+    class SortingFilter < BufferingFilter
+      include ExternalPrograms
+      def filter_all(tempfile)
+        tempfile.close
+        
+        sorted_tempfile = Tempfile.new("sort")
+        sorted_tempfile.close
+        
+        sort(tempfile, sorted_tempfile)
+        sorted_tempfile.open
+      end
+    end
+    
+    class ParsingFilter < Filter
+      def initialize(record_types)
+        @record_types = record_types
+      end
+      
+      def filter(record)
+        header_line = record
+        record_type = identify_record_type_from(header_line)
+        values = parse_values_from_subsequent_lines(record_type)
+        
+        consume_empty_line_between_records
+        
+        AsciiDataTools::Record::Record.new(record_type, values)
+      end
+      
+      protected
+      def identify_record_type_from(header_line)
+        match = header_line.match(/Record \d+ \((.*?)\)/)
+        raise "Cannot find record type in line #{header_line}!" unless match
+        expected_record_type_name = match[1]
+        @record_types.find_by_name(expected_record_type_name)
+      end
+      
+      def parse_values_from_subsequent_lines(record_type)
+        values = []
+        record_type.number_of_content_fields.times { values << parse_value_from(upstream.read) }
+        values
+      end
+      
+      def parse_value_from(line)
+        value = line.match(/.*?\[(.*?)\]/)[1]
+        value.gsub("\\n", "\n")
+      end
+      
+      def consume_empty_line_between_records
+        upstream.read
+      end
+    end
+  end
+end

data/lib/ascii-data-tools/filter/diffing.rb

@@ -0,0 +1,139 @@
+require 'ascii-data-tools/external_programs'
+
+module AsciiDataTools
+  module Filter
+    module Diffing
+      class DiffFormattingFilter < FormattingFilter
+        DIFF_DELIMITER = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n"
+      
+        def initialize(type_determiner)
+          super(nil, type_determiner)
+          def @formatter.formattable(record_type)
+            record_type.extend(Formatting::UnnumberedFormatableType) unless record_type.instance_of?(Formatting::UnnumberedFormatableType)
+          end
+        end
+      
+        def write(*streams)
+          while upstream.has_records?
+            difference = upstream.read
+            difference.left_contents.each {|rec| streams[0] << filter(rec)}
+            difference.right_contents.each {|rec| streams[1] << filter(rec)}
+            if upstream.has_records?
+              streams[0] << DIFF_DELIMITER
+              streams[1] << DIFF_DELIMITER
+            end
+          end
+        end
+      end
+    
+      class DiffExecutingFilter < BufferingFilter
+        include ExternalPrograms
+        def initialize
+          super do |tempfiles|
+            stream = diff(tempfiles)
+            raise StreamsEqualException.new if stream.eof?
+            stream
+          end
+        end
+            
+        protected
+        def upstream
+          if @first_time
+            tempfiles = @upstream.collect {|stream| buffer_as_tempfile(stream)}
+            @upstream = InputSource.new(nil, filter_all(tempfiles))
+            @first_time = false
+          end
+          @upstream
+        end
+      end
+    
+      class StreamsEqualException < Exception
+        def initialize
+          super("The streams are equal")
+        end
+      end
+
+      class Difference
+        attr_reader :left_contents, :right_contents
+      
+        def initialize(left_range, right_range)
+          @left_range, @right_range = left_range, right_range
+          @left_contents, @right_contents = [], []
+          @consumed_lines = 0
+        end
+      
+        def complete?
+          @consumed_lines == lines_to_consume
+        end
+      
+        def consume(line)
+          if line =~ /^[<>]/
+            case line[0..0]
+            when "<" then @left_contents << line[2..-1]
+            when ">" then @right_contents << line[2..-1]
+            end
+          end
+          @consumed_lines += 1
+        end
+      
+        protected
+        def left_range_length
+          @left_range.to_a.length
+        end
+      
+        def right_range_length
+          @right_range.to_a.length
+        end      
+      end
+
+      class ConflictDifference < Difference
+        protected
+        def lines_to_consume
+          left_range_length + right_range_length + 1
+        end
+      end
+
+      class AdditionDifference < Difference
+        protected
+        def lines_to_consume
+          right_range_length
+        end
+      end
+
+      class DeletionDifference < Difference
+        protected
+        def lines_to_consume
+          left_range_length
+        end
+      end
+
+      class DiffParsingFilter < Filter
+        def initialize
+          super do |record|
+            difference = make_difference_from(record)
+            difference.consume(upstream.read) until difference.complete?
+            difference
+          end
+        end
+      
+        protected
+        def make_difference_from(command_record)
+          left, command, right = split(command_record)
+          case command
+          when "c" then ConflictDifference.new(left, right)
+          when "a" then AdditionDifference.new(left, right)
+          when "d" then DeletionDifference.new(left, right)
+          else raise "Cannot parse diff line: #{command_record}"
+          end
+        end
+      
+        def split(command_record)
+          left_start, _, left_end, command, right_start, _, right_end = command_record.scan(/^(\d([,](\d))?)(c|a|d)(\d([,](\d))?)\Z/).first
+          left_range = left_start.to_i..(left_end || left_start).to_i
+          right_range = right_start.to_i..(right_end || right_start).to_i
+          [left_range, command, right_range]
+        end
+      end
+    end
+  end
+end

data/lib/ascii-data-tools/formatting.rb

@@ -0,0 +1,109 @@
+module AsciiDataTools
+  module Formatting
+    class Formatter
+      def initialize
+        @record_counter = 0
+      end
+
+      def format(record)
+        formattable(record.type).format(next_record_counter_and_increment, record.values)
+      end
+      alias :transform :format
+            
+      protected
+      def next_record_counter_and_increment
+        @record_counter += 1
+      end
+      
+      def formattable(record_type)
+        record_type.extend(FormatableType) unless record_type.instance_of?(FormatableType)
+      end
+    end
+    
+    module FormatableType
+      def format(record_number, values)
+        template % ([record_number] + escape_last(values))
+      end
+      
+      protected
+      def template
+        @template ||= [header_template, field_templates, footer].flatten.join("\n")
+      end
+      
+      def header_template
+        "Record %02d (#{name})"
+      end
+      
+      def field_templates
+        make_fields_formatable_if_necessary
+        fields.enum_with_index.map do |field, index|
+          field.template(:position => index + 1,
+                         :longest_field_name => length_of_longest_field_name,
+                         :longest_field_length => longest_field_length,
+                         :last_field? => (index + 1 == fields.length))
+        end
+      end
+      
+      def make_fields_formatable_if_necessary
+        fields.each {|field| field.extend(FormatableField) unless field.is_a?(FormatableField) }
+      end
+      
+      def longest_field_length
+        if fields.all? {|field| field.respond_to?(:length)}
+          fields.max_by {|field| field.length }.length
+        else
+          0
+        end
+      end
+      
+      def escape_last(values)
+        new_values = values.clone
+        new_values[-1] = values[-1].gsub("\n", "\\n")
+        new_values
+      end
+      
+      def footer
+        "\n"
+      end
+    end
+    
+    module UnnumberedFormatableType
+      include FormatableType
+      
+      def format(record_number, values)
+        template % escape_last(values)
+      end
+      
+      protected
+      def header_template
+        "Record (#{name})"
+      end
+    end
+    
+    module FormatableField
+      PAD_CHARACTER = "-"
+      MINIMUM_NUMBER_OF_DASHES = 5
+      def template(properties)
+        "%02d %s : [%s]%s" % [ properties[:position],
+                               padded_field_name(name, properties[:longest_field_name]),
+                               "%s",
+                               padding(properties)
+                             ]
+      end
+      
+      protected
+      def padded_field_name(field_name, length_of_longest_field_name)
+        field_name.ljust(length_of_longest_field_name)
+      end
+      
+      def padding(properties)
+        PAD_CHARACTER * number_of_pad_chars(properties)
+      end
+      
+      def number_of_pad_chars(properties)
+        return MINIMUM_NUMBER_OF_DASHES unless self.respond_to?(:length)
+        return MINIMUM_NUMBER_OF_DASHES + properties[:longest_field_length] - length - (properties[:last_field?] ? 1 : 0)
+      end
+    end
+  end
+end

data/lib/ascii-data-tools/global_autodiscovery.rb

@@ -0,0 +1,21 @@
+module AsciiDataTools
+  class << self
+    def configure(&block)
+      record_types.instance_eval(&block)
+    end
+    
+    def record_types
+      @record_types ||= RecordType::RecordTypeRepository.new
+    end
+
+    def autodiscover
+      require 'rubygems'
+
+      configuration_files_from_newest_gem_versions = Gem.find_files('ascii-data-tools/discover.rb').select do |path|
+        Gem.latest_load_paths.any? {|load_path| path.include?(load_path)} or not path.include?(Gem.default_dir)
+      end
+
+      configuration_files_from_newest_gem_versions.each {|f| load f}
+    end
+  end  
+end