rstruct 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rspec +2 -0
- data/Gemfile +13 -0
- data/LICENSE.txt +675 -0
- data/README.rdoc +137 -0
- data/Rakefile +50 -0
- data/VERSION +1 -0
- data/lib/rstruct.rb +70 -0
- data/lib/rstruct/base_types.rb +5 -0
- data/lib/rstruct/base_types/container_type.rb +102 -0
- data/lib/rstruct/base_types/packed_type.rb +78 -0
- data/lib/rstruct/base_types/type.rb +55 -0
- data/lib/rstruct/field.rb +22 -0
- data/lib/rstruct/registry.rb +66 -0
- data/lib/rstruct/struct_builder.rb +30 -0
- data/lib/rstruct/structure.rb +59 -0
- data/lib/rstruct/types.rb +44 -0
- data/samples/fatparse.rb +78 -0
- data/spec/registry_behaviors.rb +64 -0
- data/spec/registry_spec.rb +61 -0
- data/spec/rstruct_spec.rb +88 -0
- data/spec/spec_helper.rb +12 -0
- data/spec/structure_spec.rb +297 -0
- data/spec/type_behaviors.rb +158 -0
- metadata +144 -0
data/README.rdoc
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
= rstruct
|
2
|
+
|
3
|
+
Rstruct is yet another ruby binary struct parser/builder framework based around
|
4
|
+
a declarative syntax. Rstruct is designed with simplicity in mind and with a
|
5
|
+
a slightly less active focus on performance.
|
6
|
+
|
7
|
+
The goal of Rstruct is is to provide a system that lets you define structures
|
8
|
+
once, and use them in many ways as you see fit. Kind-of like... C structs in
|
9
|
+
system/API header files.
|
10
|
+
|
11
|
+
The structure declaration syntax emulates C structures syntax, although Rstruct
|
12
|
+
is not intended to be able to parse C structures from C header files. (howewver
|
13
|
+
an addon could conceivably be built to do so with minimal fuss)
|
14
|
+
|
15
|
+
Multiple approaches to parsing and building are supported and more can be added
|
16
|
+
with minimal fuss. For example, you might not wish to use the same interface to
|
17
|
+
build or parse a structure on an IO object as you would a String object. In
|
18
|
+
other words, it's nice to have something that can work easily with streamed IO
|
19
|
+
as well as buffers.
|
20
|
+
|
21
|
+
Rstruct was written because the authors wanted something comparable to C
|
22
|
+
structures without having a strong need for extra 'magic' in parsing or
|
23
|
+
building structures. While there exist numerous options in this space, they
|
24
|
+
all seem to have suffered from a combination of performance and interface issues
|
25
|
+
which limit their potential (and in some cases, spotty maintenance). Having,
|
26
|
+
tried pretty much all of these alternatives (and even contributed to a few)
|
27
|
+
in previous projects, the author still decided to write rstruct.
|
28
|
+
|
29
|
+
That said, the author does not claim Rstruct to be superior to any others, it's
|
30
|
+
just another approach. There are several other excellent binary structure
|
31
|
+
library options out there such as BinData, BitStruct, or Ruckus. Some of these
|
32
|
+
support variable length structures which, at this time, Rstruct does not. If
|
33
|
+
you are looking for something to parse variable length data automatically, you
|
34
|
+
are may be better off checking out one of these alterntives.
|
35
|
+
|
36
|
+
== Installation
|
37
|
+
|
38
|
+
(sudo)? gem install rstruct
|
39
|
+
|
40
|
+
== Synopsis
|
41
|
+
Here is a trivial example defining and packing a raw structure
|
42
|
+
|
43
|
+
require 'rstruct'
|
44
|
+
extend Rstruct::ClassMethods
|
45
|
+
|
46
|
+
example = struct(:example) {
|
47
|
+
uint32 :a
|
48
|
+
uint32 :b
|
49
|
+
uint32 :c
|
50
|
+
}.instance
|
51
|
+
|
52
|
+
example.a = 0x41
|
53
|
+
example.b = 0x42
|
54
|
+
example.c = 0x43
|
55
|
+
|
56
|
+
raw = example.write()
|
57
|
+
# => "A\x00\x00\x00B\x00\x00\x00C\x00\x00\x00" # on a little endian machine
|
58
|
+
# => "\x00\x00\x00A\x00\x00\x00B\x00\x00\x00C" # on a big endian machine
|
59
|
+
|
60
|
+
|
61
|
+
Here is a fully functional Rstruct parser example using Apple's FAT file structure.
|
62
|
+
|
63
|
+
# To compare the structs to their C counterparts, see:
|
64
|
+
# http://fxr.watson.org/fxr/source/EXTERNAL_HEADERS/mach-o/fat.h?v=xnu-1228
|
65
|
+
|
66
|
+
require 'rstruct'
|
67
|
+
extend Rstruct::ClassMethods
|
68
|
+
|
69
|
+
FAT_MAGIC = 0xcafebabe
|
70
|
+
FAT_CIGAM = 0xbebafeca
|
71
|
+
|
72
|
+
struct(:fat_header) {
|
73
|
+
uint32be :magic; # FAT_MAGIC
|
74
|
+
uint32be :nfat_arch; # number of structs that follow
|
75
|
+
}
|
76
|
+
|
77
|
+
typedef :uint32be, :cpu_type_t
|
78
|
+
typedef :uint32be, :cpu_subtype_t
|
79
|
+
|
80
|
+
struct(:fat_arch) {
|
81
|
+
cpu_type_t :cputype # cpu specifier (int)
|
82
|
+
cpu_subtype_t :cpusubtype # machine specifier (int)
|
83
|
+
uint32be :offset # file offset to this object file
|
84
|
+
uint32be :size # size of this object file
|
85
|
+
uint32be :align # alignment as a power of 2
|
86
|
+
}
|
87
|
+
|
88
|
+
# a basic helper to produce textual dumps of fields
|
89
|
+
def dump(struct)
|
90
|
+
struct.each_pair.map {|k,v| " #{k} = 0x%0.8x" % v}
|
91
|
+
end
|
92
|
+
|
93
|
+
File.open('ls.from_a_mac','r') do |f|
|
94
|
+
# Read and dump the FAT header from the file
|
95
|
+
head = get_type(:fat_header).read(f)
|
96
|
+
puts "FAT header:", dump(head)
|
97
|
+
puts
|
98
|
+
|
99
|
+
# Read and dump the architectures after the header
|
100
|
+
|
101
|
+
fat_arch = get_type(:fat_arch)
|
102
|
+
(head.nfat_arch).times do |i|
|
103
|
+
arch = fat_arch.read(f) # note, it reads to a seperate object on each arch
|
104
|
+
puts " Architecture #{i}:"
|
105
|
+
puts " " << dump(arch).join("\n ")
|
106
|
+
puts
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
.. which should produce output something like the following:
|
111
|
+
|
112
|
+
FAT header:
|
113
|
+
magic = 0xcafebabe
|
114
|
+
nfat_arch = 0x00000002
|
115
|
+
|
116
|
+
Architecture 0:
|
117
|
+
cputype = 0x01000007
|
118
|
+
cpusubtype = 0x80000003
|
119
|
+
offset = 0x00001000
|
120
|
+
size = 0x00009ab0
|
121
|
+
align = 0x0000000c
|
122
|
+
|
123
|
+
Architecture 1:
|
124
|
+
cputype = 0x00000007
|
125
|
+
cpusubtype = 0x00000003
|
126
|
+
offset = 0x0000b000
|
127
|
+
size = 0x00008b30
|
128
|
+
align = 0x0000000c
|
129
|
+
|
130
|
+
|
131
|
+
Please refer to rdoc, the samples/ directory, and unit tests for more information.
|
132
|
+
|
133
|
+
== Copyright
|
134
|
+
|
135
|
+
Copyright (c) 2011 Eric Monti. See LICENSE.txt for
|
136
|
+
further details.
|
137
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'rake'
|
11
|
+
|
12
|
+
require 'jeweler'
|
13
|
+
Jeweler::Tasks.new do |gem|
|
14
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
15
|
+
gem.name = "rstruct"
|
16
|
+
gem.homepage = "http://github.com/emonti/rstruct"
|
17
|
+
gem.license = "GPL"
|
18
|
+
gem.summary = %Q{A library for working with Ruby binary structures in a way similar to c-structs}
|
19
|
+
gem.description = %Q{A library for working with Ruby binary structures in a way similar to c-structs}
|
20
|
+
gem.email = "esmonti@gmail.com"
|
21
|
+
gem.authors = ["Eric Monti"]
|
22
|
+
# Include your dependencies below. Runtime dependencies are required when using your gem,
|
23
|
+
# and development dependencies are only needed for development (ie running rake tasks, tests, etc)
|
24
|
+
# gem.add_runtime_dependency 'jabber4r', '> 0.1'
|
25
|
+
# gem.add_development_dependency 'rspec', '> 1.2.3'
|
26
|
+
end
|
27
|
+
Jeweler::RubygemsDotOrgTasks.new
|
28
|
+
|
29
|
+
require 'rspec/core'
|
30
|
+
require 'rspec/core/rake_task'
|
31
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
32
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
33
|
+
end
|
34
|
+
|
35
|
+
RSpec::Core::RakeTask.new(:rcov) do |spec|
|
36
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
37
|
+
spec.rcov = true
|
38
|
+
end
|
39
|
+
|
40
|
+
task :default => :spec
|
41
|
+
|
42
|
+
require 'rake/rdoctask'
|
43
|
+
Rake::RDocTask.new do |rdoc|
|
44
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
45
|
+
|
46
|
+
rdoc.rdoc_dir = 'rdoc'
|
47
|
+
rdoc.title = "rstruct #{version}"
|
48
|
+
rdoc.rdoc_files.include('README*')
|
49
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
50
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/lib/rstruct.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'rstruct/registry'
|
3
|
+
require 'rstruct/types'
|
4
|
+
require 'rstruct/structure'
|
5
|
+
require 'rstruct/struct_builder'
|
6
|
+
|
7
|
+
module Rstruct
|
8
|
+
module ClassMethods
|
9
|
+
|
10
|
+
# Take the following C struct for example:
|
11
|
+
#
|
12
|
+
# struct mach_header {
|
13
|
+
# uint32_t magic; /* mach magic number identifier */
|
14
|
+
# cpu_type_t cputype; /* cpu specifier */
|
15
|
+
# cpu_subtype_t cpusubtype; /* machine specifier */
|
16
|
+
# uint32_t filetype; /* type of file */
|
17
|
+
# uint32_t ncmds; /* number of load commands */
|
18
|
+
# uint32_t sizeofcmds; /* the size of all the load commands */
|
19
|
+
# uint32_t flags; /* flags */
|
20
|
+
# }
|
21
|
+
#
|
22
|
+
# Which might be defined with rstruct as follows:
|
23
|
+
#
|
24
|
+
# extend(Rstruct::ClassMethods)
|
25
|
+
#
|
26
|
+
# typedef :uint32_t, :cpu_type_t
|
27
|
+
# typedef :uint32_t, :cpu_subtype_t
|
28
|
+
#
|
29
|
+
# struct(:mach_header) {
|
30
|
+
# uint32_t :magic # mach magic number identifier
|
31
|
+
# cpu_type_t :cputype # cpu specifier
|
32
|
+
# cpu_subtype_t :cpusubtype # machine specifier
|
33
|
+
# uint32_t :filetype # type of file
|
34
|
+
# uint32_t :ncmds # number of load commands
|
35
|
+
# uint32_t :sizeofcmds # the size of all the load commands
|
36
|
+
# uint32_t :flags # flags
|
37
|
+
# }
|
38
|
+
#
|
39
|
+
def struct(name, opts={},&block)
|
40
|
+
Rstruct::Structure.new(name, opts, &block)
|
41
|
+
end
|
42
|
+
|
43
|
+
def typedef(p, t, opts={})
|
44
|
+
reg = opts[:registry] || default_registry
|
45
|
+
reg.typedef(p,t,opts)
|
46
|
+
end
|
47
|
+
|
48
|
+
def sizeof(typ, reg=nil)
|
49
|
+
reg ||= default_registry
|
50
|
+
if t=reg[typ]
|
51
|
+
t.sizeof
|
52
|
+
else
|
53
|
+
raise(InvalidTypeError, "unknown type: #{typ}")
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns the default Rstruct registry
|
58
|
+
def default_registry
|
59
|
+
Registry::DEFAULT_REGISTRY
|
60
|
+
end
|
61
|
+
|
62
|
+
def get_type(typ, reg=Registry::DEFAULT_REGISTRY)
|
63
|
+
reg[typ]
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
extend(ClassMethods)
|
69
|
+
end
|
70
|
+
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'rstruct/base_types/type'
|
2
|
+
|
3
|
+
module Rstruct
|
4
|
+
|
5
|
+
module ContainerMixins
|
6
|
+
def rstruct_type
|
7
|
+
@rstruct_type
|
8
|
+
end
|
9
|
+
|
10
|
+
def rstruct_type=(val)
|
11
|
+
if @rstruct_type
|
12
|
+
raise(ArgumentError, "Can't override the rstruct_type once it is set")
|
13
|
+
else
|
14
|
+
@rstruct_type = val
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def write(dst=nil, pvals=nil)
|
19
|
+
if dst.is_a?(String)
|
20
|
+
l = dst.size
|
21
|
+
dst = StringIO.new(dst)
|
22
|
+
dst.pos = l
|
23
|
+
elsif dst.nil?
|
24
|
+
dst = StringIO.new
|
25
|
+
end
|
26
|
+
|
27
|
+
typ ||= self.rstruct_type
|
28
|
+
|
29
|
+
|
30
|
+
vals = (pvals.respond_to?(:values) ? pvals.values : pvals)
|
31
|
+
vals ||= self.values
|
32
|
+
|
33
|
+
opos = dst.pos
|
34
|
+
typ.fields.each_with_index do |f, i|
|
35
|
+
fldval = vals[i]
|
36
|
+
if fldval.respond_to?(:write)
|
37
|
+
fldval.write(dst, fldval)
|
38
|
+
else
|
39
|
+
dst.write(f.typ.pack_value(fldval, self))
|
40
|
+
end
|
41
|
+
end
|
42
|
+
if dst.is_a?(StringIO) and pvals.nil?
|
43
|
+
dst.pos = opos
|
44
|
+
return(dst.read)
|
45
|
+
else
|
46
|
+
return dst.pos - opos
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class ContainerType < Type
|
52
|
+
include Packable
|
53
|
+
|
54
|
+
def initialize(*args, &block)
|
55
|
+
@countainer = true
|
56
|
+
super(*args, &block)
|
57
|
+
end
|
58
|
+
|
59
|
+
def groupable?
|
60
|
+
self.fields.find {|f| not f.groupable? }.nil?
|
61
|
+
end
|
62
|
+
|
63
|
+
def format
|
64
|
+
self.fields.map do |f|
|
65
|
+
if f.groupable?
|
66
|
+
f.format
|
67
|
+
else
|
68
|
+
return nil
|
69
|
+
end
|
70
|
+
end.join
|
71
|
+
end
|
72
|
+
|
73
|
+
def sizeof
|
74
|
+
self.fields.inject(0) do |s,v|
|
75
|
+
if vs=v.typ.sizeof
|
76
|
+
s+=vs
|
77
|
+
else
|
78
|
+
return nil
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def field_names
|
84
|
+
@field_names ||= self.fields.map{|f| f.name }
|
85
|
+
end
|
86
|
+
|
87
|
+
def field_types
|
88
|
+
@field_types ||= self.fields.map{|f| f.typ }
|
89
|
+
end
|
90
|
+
|
91
|
+
def read(raw, obj=nil)
|
92
|
+
raw = StringIO.new(raw) if raw.is_a?(String)
|
93
|
+
obj = self.instance()
|
94
|
+
fields.each do |f|
|
95
|
+
obj[f.name] = f.read(raw, obj)
|
96
|
+
end
|
97
|
+
return obj
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'rstruct/base_types/type'
|
2
|
+
|
3
|
+
module Rstruct
|
4
|
+
class PackError < StandardError
|
5
|
+
end
|
6
|
+
|
7
|
+
class ReadError < StandardError
|
8
|
+
end
|
9
|
+
|
10
|
+
module Packable
|
11
|
+
private
|
12
|
+
# sets up a callback for packing a value to raw data
|
13
|
+
# for this type
|
14
|
+
def on_pack(&block)
|
15
|
+
@pack_cb = block
|
16
|
+
end
|
17
|
+
|
18
|
+
# sets up a callback for unpacking data from a string for
|
19
|
+
# this type
|
20
|
+
def on_unpack(&block)
|
21
|
+
@unpack_cb = block
|
22
|
+
end
|
23
|
+
|
24
|
+
public
|
25
|
+
# Called when parsing. While you can override this in subclasses,
|
26
|
+
# in general it is probably better to use the 'on_unpack' method
|
27
|
+
# to define a proc to handle unpacking for special cases.
|
28
|
+
def read(raw, predecessors=nil)
|
29
|
+
if raw.respond_to?(:read)
|
30
|
+
raw = raw.read(self.sizeof())
|
31
|
+
end
|
32
|
+
if raw.size < self.sizeof()
|
33
|
+
raise(ReadError, "Expected #{self.sizeof} bytes, but only got #{raw.size} bytes")
|
34
|
+
end
|
35
|
+
|
36
|
+
vals =
|
37
|
+
if @unpack_cb
|
38
|
+
@unpack_cb.call(raw, predecessors)
|
39
|
+
else
|
40
|
+
raw.unpack(self.format)
|
41
|
+
end
|
42
|
+
return(self.claim_value(vals, predecessors))
|
43
|
+
end
|
44
|
+
|
45
|
+
# Called when composing raw data. While you can override this in
|
46
|
+
# subclasses, in general it is probably better to use the 'on_pack'
|
47
|
+
# method to define a proc to handle packing for special cases.
|
48
|
+
def pack_value(val, obj=nil)
|
49
|
+
begin
|
50
|
+
if @pack_cb
|
51
|
+
@pack_cb.call(val, obj)
|
52
|
+
else
|
53
|
+
varray = val.is_a?(Array) ? val : [val]
|
54
|
+
varray.pack(self.format)
|
55
|
+
end
|
56
|
+
rescue => e
|
57
|
+
raise(PackError, "Error packing #{val.inspect} as type #{self.name.inspect} -- #{e.class} -> #{e}")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class PackedType < Type
|
63
|
+
include Packable
|
64
|
+
attr_reader :size, :format
|
65
|
+
|
66
|
+
def initialize(name, size, format, opts={}, &block)
|
67
|
+
@size = size
|
68
|
+
@format = format
|
69
|
+
@groupable = true
|
70
|
+
super(name, opts, &block)
|
71
|
+
end
|
72
|
+
|
73
|
+
def instance(val=nil)
|
74
|
+
val
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|