px4_log_reader 0.0.5

data/lib/px4_log_reader/message_descriptor_cache.rb

@@ -0,0 +1,56 @@
+module Px4LogReader
+
+	class MessageDescriptorCache
+
+		attr_reader :cache_filename
+
+		def initialize( filename )
+			@cache_filename = filename
+		end
+
+		def exist?
+			return File.exist?( @cache_filename )
+		end
+
+		def read_descriptors
+
+			message_descriptors = {}
+
+         if File.exist?( cache_filename )
+            File.open( cache_filename, 'r' ) do |input|
+               begin
+               	while ( ( data = input.read(4) ) && ( data.length == 4 ) ) do
+                     descriptor_size = data.unpack('L').first
+                     descriptor = Marshal.load( input.read( descriptor_size ) )
+
+                     message_descriptors[ descriptor.type ] = descriptor
+                  end
+               rescue EOFError => error
+                  puts "Parsed #{@message_descriptions.size} cached message descriptions"
+               rescue StandardError => error
+                  puts "#{error.class}: #{error.message}"
+                  puts error.backtrace.join("\n")
+               end
+            end
+         else
+            puts "Cache file '#{cache_filename}' not found"
+         end
+
+         return message_descriptors
+		end
+
+		def write_descriptors( message_descriptors )
+			if !@cache_filename.empty? && File.exist?( File.dirname( @cache_filename ) )
+				File.open( @cache_filename, 'w+' ) do |output|
+					message_descriptors.each do |message_type,descriptor|
+						descriptor_data = Marshal.dump( descriptor )
+						output.write( [ descriptor_data.size ].pack('L') )
+						output.write( descriptor_data )
+					end
+				end
+			end
+		end
+
+	end
+
+end

data/lib/px4_log_reader/reader.rb

@@ -0,0 +1,160 @@
+module Px4LogReader	
+
+	def self.open_common( file, options, &block )
+
+		reader = Reader.new( file, options )
+
+		yield reader if block_given?
+
+		return reader
+	end
+
+	def self.open( filename, options = {}, &block  )
+
+		reader = nil
+
+		if File.exist?( filename )
+			reader = self.open_common( File.open( filename, 'r' ), options, &block )
+		end	
+
+		return reader
+
+	end
+
+	def self.open!( filename, options = {}, &block )
+		reader = nil
+
+		if File.exist?( filename )
+			reader = self.open_common( File.open( filename, 'r' ), options, &block )
+		else
+			raise FileNotFoundError.new( filename )
+		end	
+
+		return reader
+	end
+
+   class Context
+   	attr_reader :messages
+   	def initialize
+   		messages = {}
+   	end
+   	def find_by_name( name )
+   		named_message = nil
+   		@messages.values.each do |message|
+   			if message.descriptor.name == name
+   				named_message = message
+   			end
+   		end
+   		return named_message
+   	end
+   	def find_by_type( type )
+   		return @messages[ type ]
+   	end
+   	def set( message )
+   		@messages[ message.descriptor.type ] = message.dup
+   	end
+   end
+
+	class Reader   
+
+		def initialize( file, options )
+
+			opts = {
+				cache_filename: '',
+				buffer_size_kb: 10 * 1024
+			}.merge( options )
+
+			@message_descriptors = {}
+			@buffers = LogBufferArray.new
+			@descriptor_cache = nil
+			@context = Context.new
+
+			@log_file = file
+			# @buffers.set_file( @log_file, load_buffers: true )
+
+			@descriptor_cache = MessageDescriptorCache.new( opts[:cache_filename] )
+		end
+
+		def descriptors
+			if @log_file && @message_descriptors.empty?
+				if @descriptor_cache && @descriptor_cache.exist?
+					@message_descriptors = @descriptor_cache.read_descriptors
+				else
+					@message_descriptors = LogFile::read_descriptors( @log_file, @descriptor_cache )
+				end
+
+				@message_descriptors[ FORMAT_MESSAGE.type ] = FORMAT_MESSAGE
+			end
+
+			return @message_descriptors
+		end
+
+		def each_message( options = {}, &block )
+
+			opts ={
+				with: [],        # white list - empty means all minus those in without list
+				without: ['FMT'] # black list - includes types or names
+			}.merge( options || {} )
+
+			opts[:with].map! do |val|
+				if val.class == String
+					descriptor = descriptors.values.find { |desc| desc.name == val }
+					val = descriptor.type
+				end
+			end
+
+			opts[:without].map! do |val|
+				if val.class == String
+					descriptor = descriptors.values.find { |desc| desc.name == val }
+
+					if descriptor
+						val = descriptor.type
+					else
+						raise "Failed to find descriptor with name '#{val}'"
+					end
+				end
+			end
+
+			if block_given?
+
+				loop do
+
+					message = LogFile::read_message( @log_file, @message_descriptors )
+					break if message.nil?
+
+					# Added message to the set of latest messages.
+					@context.set( message )
+
+					if opts[:with].empty?
+						if !opts[:without].include?( message.descriptor.name )
+							yield message, @context
+						end
+					else
+						if opts[:with].include?( message.descriptor.type )
+							yield message, @context
+						end
+					end
+
+				end
+
+			else
+				raise BlockRequiredError.new
+			end
+
+		end
+
+		# def rewind
+		# 	if @log_file
+
+		# 	@log_file.rewind
+		# 	@buffers.load_buffers
+
+		# 	end
+		# end
+
+		# def seek( offset )
+		# end
+
+	end
+
+end

data/lib/px4_log_reader/version.rb

@@ -0,0 +1,3 @@
+module Px4LogReader
+	VERSION = '0.0.5'
+end

data/lib/px4_log_reader.rb

@@ -0,0 +1,8 @@
+require 'px4_log_reader/version'
+require 'px4_log_reader/invalid_descriptor_error'
+require 'px4_log_reader/log_message'
+require 'px4_log_reader/message_descriptor'
+require 'px4_log_reader/log_buffer'
+require 'px4_log_reader/log_file'
+require 'px4_log_reader/message_descriptor_cache'
+require 'px4_log_reader/reader'

data/lib/px4log_parser.rb

@@ -0,0 +1,228 @@
+
+
+
+
+# class Px4LogReader
+
+#    include PX4
+
+#    HEADER_MARKER = [0xA3,0x95]#.pack('CC').freeze
+#    HEADER_LENGTH = 3
+
+#    FORMAT_MESSAGE = Px4LogMessageDescription.new({
+#       name: 'FMT',
+#       type: 0x80,
+#       length: 89,
+#       format: 'BBnZ',
+#       fields: ["Type", "Length", "Name", "Format", "Labels"] }).freeze
+
+#    attr_reader :message_descriptions
+#    attr_reader :messages
+#    attr_reader :px4_log_format
+
+#    def initialize
+#       @message_descriptions = {}
+#       @messages = []
+#       @px4_log_format = false
+#    end
+
+#    def parse_log( filename, cache_filename=nil )
+#       @message_descriptions = {}
+#       @messages = []
+
+#       @file_size = File.size?(filename)
+
+#       if cache_filename && File.exist?( cache_filename )
+#          if File.exist?( cache_filename )
+#             File.open( cache_filename, 'r' ) do |io|
+#                begin
+#                   loop do
+#                      description_size = io.read_nonblock(4).unpack('L').first
+#                      description = Marshal.load( io.read(description_size) )
+
+#                      @message_descriptions[ description.type ] = description
+#                   end
+#                rescue EOFError => error
+#                   puts "Parsed #{@message_descriptions.size} cached message descriptions"
+#                   # @message_descriptions.each do |message_type,description|
+#                   #    puts description
+#                   # end
+#                rescue StandardError => error
+#                   puts "#{error.class}: #{error.message}"
+#                   puts error.backtrace.join("\n")
+#                end
+#             end
+#          else
+#             puts "Cache file '#{cache_filename}' not found"
+#          end
+#       else
+#          File.open( filename, 'r' ) do |io|
+#             read_formats( io, cache_filename )
+#          end
+#       end
+
+#       if @message_descriptions.size > 0
+#          File.open( filename, 'r' ) do |io|
+#             begin
+#                loop do
+#                   message = read_message( io )
+
+#                   if message.nil?
+#                      puts "Failed to read message"
+#                      break
+#                   elsif message.description.name != "FMT"
+#                      @messages << message
+#                   end
+
+#                   $stdout.printf "\rReading messages %d/%d", io.pos, @file_size
+#                end
+#             rescue StandardError => error
+#                puts "#{error.class}: #{error.message}"
+#                puts error.backtrace.join("\n")
+#             end
+#          end
+#       else
+#          raise "No message descriptions found"
+#       end
+#       puts
+#    end
+
+#    def read_message_header( io )
+#       byte = nil
+
+#       begin
+
+#          data = io.read(2)
+
+#          if data && data.length == 2
+#             loop do
+
+#                data << io.read_nonblock(1)
+
+#                # puts "#{data.unpack('CCC')[0,2]} == #{HEADER_MARKER}"
+#                if data.unpack('CCC')[0,2] == HEADER_MARKER
+#                   byte = data.unpack('CCC').last & 0xFF
+#                   # puts "Found message header #{data.unpack('CCC')}"
+#                   # puts "message_type = #{'%02X'%byte}"
+#                   break
+#                else
+#                   data = data[1..-1]
+#                end
+
+#             end
+#          end
+
+#       rescue EOFError => error
+#          # Nothing to do.
+#       rescue StandardError => error
+#          puts error.message
+#          puts error.backtrace.join("\n")
+#       end
+
+#       return byte
+#    end
+
+#    def read_message( io )
+
+#       message = nil
+#       while message.nil? do
+#          message_type = read_message_header( io )
+
+#          if message_type && (message_type != FORMAT_MESSAGE.type)
+#             message_description = @message_descriptions[ message_type ]
+
+#             if message_description
+#                message_data = io.read( message_description.length - HEADER_LENGTH )
+#                message = message_description.parse_message( message_data )
+#             else
+#                puts "ERROR: Failed to get description for message of type '#{'0x%02X' % message_type}'"
+#             end
+#          elsif message_type.nil?
+#             break
+#          end
+
+#          # $stdout.printf "\rReading formats %d/%d", io.pos, @file_size
+#       end
+
+#       return message
+#    end
+
+#    def read_formats( io, cache_filename )
+#       loop do
+#          begin
+#             message_type = read_message_header( io )
+
+#             if message_type.nil?
+#                break
+#             elsif message_type == FORMAT_MESSAGE.type
+#                # puts "Found format message"
+#                message_description = Px4LogMessageDescription.new
+#                message_description.parse_from_io( io )
+
+#                unless @message_descriptions.keys.include? message_description.type
+#                   @message_descriptions[message_description.type] = message_description
+#                end
+
+#                if message_description.name == "TIME"
+#                   @px4_log_format = true
+#                end
+#             end
+
+#             $stdout.printf "\rReading formats %d/%d", io.pos, @file_size
+
+#          rescue StandardError => e
+#             puts "#{e.class}: #{e.message}"
+#             puts e.backtrace.join("\n")
+#             break
+#          end
+#       end
+#       $stdout.puts
+
+#       if cache_filename
+#          File.open( cache_filename, 'w+' ) do |io|
+#             @message_descriptions.each do |message_type,description|
+#                description_data = Marshal.dump( description )
+#                io.write( [ description_data.size ].pack('L') )
+#                io.write( description_data )
+#             end
+#          end
+#       end
+#    end
+
+# end
+
+
+# if ARGV.size == 2
+#    filename = ARGV[0]
+#    output_dir = ARGV[1]
+#    puts "Attempting to parse #{filename}"
+
+#    cache_filename = File.join( 'px4_csvs', "#{File.basename( filename, '.px4log' )}.px4log_description_cache" )
+
+#    log = Px4LogReader.new
+#    log.parse_log( filename, cache_filename )
+
+#    log.message_descriptions.each do |type,description|
+#       # puts description
+#       File.open(File.join(output_dir,"#{description.name.downcase}_log.csv"),"w+") do |io|
+#          io.puts description.to_csv_line
+#       end
+#    end
+
+#    last_timestamp = 0
+#    log.messages.each do |message|
+#       if message.description.name == "TIME"
+#          last_timestamp = message.get(0)
+#       end
+
+#       File.open(File.join(output_dir,"#{message.description.name.downcase}_log.csv"),"a+") do |io|
+#          io.puts message.to_csv_line(last_timestamp)
+#       end
+#    end
+
+# else
+
+#    puts "Specify source and destination"
+
+# end
+

data/px4_log_reader.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+require_relative 'lib/px4_log_reader/version'
+
+Gem::Specification.new do |s|
+   s.name        = 'px4_log_reader'
+   s.version     = Px4LogReader::VERSION
+   s.date        = Time.now.to_date.strftime('%Y-%m-%d')
+   s.summary     = "PX4 flight log reader"
+   s.description = "Px4LogReader is a gem for parsing PX4-format flight logs generated by the Pixhawk."
+   s.authors     = [ "Robert Glissmann" ]
+   s.email       = 'Robert.Glissmann@gmail.com'
+   s.files       = `git ls-files`.split("\n")
+   s.licenses    = ['BSD']
+   s.homepage    = 'https://github.com/rgmann/px4_log_reader'
+
+   s.add_development_dependency 'rspec', '~> 3.1'
+end

data/test/test.rb

@@ -0,0 +1,33 @@
+if ARGV.size == 2
+   filename = ARGV[0]
+   output_dir = ARGV[1]
+   puts "Attempting to parse #{filename}"
+
+   cache_filename = File.join( 'px4_csvs', "#{File.basename( filename, '.px4log' )}.px4log_description_cache" )
+
+   log = Px4LogReader.new
+   log.parse_log( filename, cache_filename )
+
+   log.message_descriptions.each do |type,description|
+      # puts description
+      File.open(File.join(output_dir,"#{description.name.downcase}_log.csv"),"w+") do |io|
+         io.puts description.to_csv_line
+      end
+   end
+
+   last_timestamp = 0
+   log.messages.each do |message|
+      if message.description.name == "TIME"
+         last_timestamp = message.get(0)
+      end
+
+      File.open(File.join(output_dir,"#{message.description.name.downcase}_log.csv"),"a+") do |io|
+         io.puts message.to_csv_line(last_timestamp)
+      end
+   end
+
+else
+
+   puts "Specify source and destination"
+
+end

data/test/test_log_buffer.rb

@@ -0,0 +1,173 @@
+require 'minitest/autorun'
+require 'px4_log_reader'
+
+class TestLogBuffer < MiniTest::Test
+
+	def setup
+	end
+
+	def test_buffer_create
+
+		buffer_size = 256
+		buffer = Px4LogReader::LogBuffer.new( buffer_size )
+
+		assert_equal buffer_size, buffer.data.size
+		assert_equal Array.new( buffer_size, 0x00 ), buffer.data
+		assert_equal 0, buffer.read_position
+		assert_equal 0, buffer.write_position
+		assert_equal true, buffer.empty?
+
+	end
+
+	def test_buffer_read_write
+
+		buffer_size = 256
+		buffer = Px4LogReader::LogBuffer.new( buffer_size )
+
+		test_filename = './test_buffer_read_write.bin'
+		generate_test_file( test_filename, buffer_size )
+
+		test_file = File.open( test_filename,'rb+')
+		
+		assert_equal true, buffer.empty?
+
+		buffer.write( test_file )
+		assert_equal 0, buffer.read_position
+		assert_equal 256, buffer.write_position
+		assert_equal false, buffer.empty?
+
+		test_read_1_size = 112
+		data = buffer.read( test_read_1_size )
+		assert_equal test_read_1_size, data.size
+		assert_equal test_read_1_size, buffer.read_position
+		assert_equal buffer_size, buffer.write_position
+		assert_equal false, buffer.empty?
+
+		test_read_2_size = 256
+		data = buffer.read( test_read_2_size )
+		assert_equal (buffer_size - test_read_1_size), data.size
+		assert_equal buffer_size, buffer.read_position
+		assert_equal buffer_size, buffer.write_position
+		assert_equal true, buffer.empty?
+
+		test_file.close
+
+		FileUtils.rm test_filename
+
+	end
+
+	def test_buffer_array
+
+		buffer_array = Px4LogReader::LogBufferArray.new
+
+		file_size = 256
+		test_filename = './test_buffer_array.bin'
+		generate_test_file( test_filename, file_size )
+
+		buffer_size  = file_size
+		buffer_count = 4
+		buffer_array.set_file( File.open( test_filename, 'rb' ), buffer_size: buffer_size, buffer_count: buffer_count )
+
+		assert_equal buffer_count, buffer_array.buffers.count
+		assert_equal 0, buffer_array.current_buffer_index
+
+		buffer_array.buffers.each do |buffer|
+			assert_equal false, buffer.empty?
+			assert_equal 0, buffer.read_position
+			assert_equal (buffer_size / buffer_count), buffer.write_position
+			assert_equal (buffer_size / buffer_count), buffer.data.size
+		end
+
+		file_size.times do |index|
+			assert_equal index, buffer_array.read(1).unpack('C').first
+		end
+
+		FileUtils.rm test_filename
+
+	end
+
+
+	def test_buffer_array_refill
+		buffer_array = Px4LogReader::LogBufferArray.new
+
+		file_size = 256
+		test_filename = './test_buffer_array_refill.bin'
+		generate_test_file( test_filename, file_size )
+
+		buffer_size  = file_size / 2
+		buffer_count = 4
+		single_buffer_size = buffer_size / buffer_count
+		buffer_array.set_file( File.open( test_filename, 'rb' ), buffer_size: buffer_size, buffer_count: buffer_count )
+
+		assert_equal buffer_count, buffer_array.buffers.count
+		assert_equal 0, buffer_array.current_buffer_index
+
+		buffer_array.buffers.each do |buffer|
+			assert_equal false, buffer.empty?
+			assert_equal (buffer_size / buffer_count), buffer.write_position
+			assert_equal (buffer_size / buffer_count), buffer.data.size
+		end
+
+		data = ''
+
+		# Read enough data to empty the first buffer.
+		data << buffer_array.read( single_buffer_size + single_buffer_size / 2 )
+		assert buffer_array.buffers[0].empty?
+
+		# Refill the empty buffer.
+		buffer_array.load_empty_buffers
+		assert_equal false, buffer_array.buffers[0].empty?
+		data << buffer_array.read( single_buffer_size / 2 + 3 * single_buffer_size )
+		assert_equal 5 * single_buffer_size, data.size
+
+		# Verify that we cannot read any more data.
+		assert_equal '', buffer_array.read( single_buffer_size )
+
+		assert_equal true, validate_data( test_filename, data )
+
+
+		FileUtils.rm test_filename
+	end
+
+
+	def generate_test_file( path, size )
+		test_filename = './test_buffer_read_write.bin'
+		File.open( path, 'wb+' ) do |io|
+			value = 0
+			size.times do
+				io.write( [ value ].pack('C') )
+				value = ( value + 1 ) % 256
+			end
+		end
+	end
+
+	def validate_data( path, data, max_bytes = nil )
+		equal = true
+
+		max_bytes = data.size if max_bytes.nil?
+
+		data = data.unpack('C*')
+
+		File.open( path, 'rb' ) do |io|
+			comp_index = 0
+			while ( comp_index < max_bytes ) && equal do
+				begin
+					byte = io.read(1).unpack('C').first
+					if data[comp_index] != byte
+						puts "mismatch at offset=#{comp_index}: expected '#{"0x%02X"%byte}; found '#{"0x%02X"%data[comp_index]}'"
+						equal = false
+					end
+				rescue EOFError => error
+					puts "reached EOF before end of data"
+					equal = false
+				end
+				comp_index += 1
+			end
+		end
+
+		return equal
+	end
+
+end
+
+