binary_parser 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +239 -0
- data/Rakefile +11 -0
- data/binary_parser.gemspec +33 -0
- data/lib/binary_parser.rb +50 -0
- data/lib/binary_parser/version.rb +3 -0
- data/lib/built_in_template/binary.rb +12 -0
- data/lib/built_in_template/flag.rb +22 -0
- data/lib/built_in_template/uint.rb +9 -0
- data/lib/error.rb +21 -0
- data/lib/general_class/abstract_binary.rb +68 -0
- data/lib/general_class/bit_position.rb +22 -0
- data/lib/general_class/condition.rb +11 -0
- data/lib/general_class/expression.rb +66 -0
- data/lib/loop_list.rb +52 -0
- data/lib/nameless_template.rb +7 -0
- data/lib/scope.rb +87 -0
- data/lib/stream_template_base.rb +129 -0
- data/lib/structure_definition.rb +131 -0
- data/lib/template_base.rb +120 -0
- data/unit_test/general_class/test_abstract_binary.rb +60 -0
- data/unit_test/general_class/test_bit_position.rb +22 -0
- data/unit_test/general_class/test_condition.rb +23 -0
- data/unit_test/general_class/test_expression.rb +22 -0
- data/unit_test/test_scope.rb +191 -0
- data/unit_test/test_stream_template_base.rb +232 -0
- data/unit_test/test_structure_definition.rb +115 -0
- data/unit_test/test_template_base.rb +62 -0
- metadata +108 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2d5b2692b4a9c6b2bb5be201074f80779039f881
|
4
|
+
data.tar.gz: 0e6c1da7ab3aa2ca1f8e79150c34262ec08a4772
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1c59d35586e0f044ebdef155ef191169a4cf3dd65ba2743d9c19974759aefd55e27d93bb1121443a81c350617b7b75ca17cc9b6d9e05de522f4e9e300f650e7b
|
7
|
+
data.tar.gz: def0d9f155a7af845d78ce08b87555acca72c11e1f63ae41b716c0fcbc94536e8d66148d1946d2be9e174ec648b0948ebe67df2216be9199d0c9d2a727aa9259
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 rokugats(u)
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,239 @@
|
|
1
|
+
Ruby-Binary-Parser
|
2
|
+
===================
|
3
|
+
Ruby-Binary-Parser is Ruby Gem library providing DSL for parsing binary-data, such as Image files, Video files, etc.
|
4
|
+
Without operating bytes and bits complicatedly, you can parse and read binary-data *generically* and *abstractly*.
|
5
|
+
|
6
|
+
|
7
|
+
Description
|
8
|
+
-----------
|
9
|
+
This library can parse all kind of binary-data structures including non-fixed length of structures and nested structures.
|
10
|
+
For generic parsing, loop and condition(if) statement to define structures is provided in this library.
|
11
|
+
Of course, values of neighbor binary-data can be used as the other binary-data's specification of length.
|
12
|
+
|
13
|
+
Furthermore, this library handles all binary-data under the lazy evaluation.
|
14
|
+
So you can read required parts of a binary-data very quickly even if whole of the binary-data is too big,
|
15
|
+
|
16
|
+
|
17
|
+
Notice
|
18
|
+
------
|
19
|
+
Currently, this library supports only READ of binary-data.
|
20
|
+
So you cannot WRITE binary-data directly with this library.
|
21
|
+
|
22
|
+
|
23
|
+
Usage
|
24
|
+
-----
|
25
|
+
Look at following examples to quickly understand how to use this library.
|
26
|
+
|
27
|
+
### Install ###
|
28
|
+
$ gem install binary_parser
|
29
|
+
|
30
|
+
|
31
|
+
### Example 1 ###
|
32
|
+
Consider the following (temporary) binary structures which describe Image data.
|
33
|
+
|
34
|
+
<table style="margin-left:auto;margin-right:auto;">
|
35
|
+
<tr><td colspan=4 style="text-align:center;">MyImage (non-fixed length)</td></tr>
|
36
|
+
<tr style="background-color:lightsteelblue;">
|
37
|
+
<td style="width:150px;">Data Name</td>
|
38
|
+
<td style="width:80px;">Type</td>
|
39
|
+
<td style="width:100px;">Bit Length</td>
|
40
|
+
<td style="width:200px;">Number Of Replications</td>
|
41
|
+
</tr>
|
42
|
+
<tr>
|
43
|
+
<td>height</td>
|
44
|
+
<td>UInt</td>
|
45
|
+
<td>8</td>
|
46
|
+
<td>1</td>
|
47
|
+
</tr>
|
48
|
+
<tr>
|
49
|
+
<td>width</td>
|
50
|
+
<td>UInt</td>
|
51
|
+
<td>8</td>
|
52
|
+
<td>1</td>
|
53
|
+
</tr>
|
54
|
+
<tr>
|
55
|
+
<td>RGB color bit-map</td>
|
56
|
+
<td>UInt</td>
|
57
|
+
<td>8 * 3</td>
|
58
|
+
<td>'height' * 'width'</td>
|
59
|
+
</tr>
|
60
|
+
<tr>
|
61
|
+
<td>has date?</td>
|
62
|
+
<td>Flag</td>
|
63
|
+
<td>1</td>
|
64
|
+
<td>1</td>
|
65
|
+
</tr>
|
66
|
+
<tr>
|
67
|
+
<td>date</td>
|
68
|
+
<td>MyDate</td>
|
69
|
+
<td>31</td>
|
70
|
+
<td>'has date?' is 1 => 1<br>else => 0</td>
|
71
|
+
</tr>
|
72
|
+
</table>
|
73
|
+
|
74
|
+
|
75
|
+
<table style="margin-left:auto;margin-right:auto;">
|
76
|
+
<tr><td colspan=4 style="text-align:center;">MyDate (31 bit)</td></tr>
|
77
|
+
<tr style="background-color:lightsteelblue;">
|
78
|
+
<td style="width:150px;">Data Name</td>
|
79
|
+
<td style="width:80px;">Type</td>
|
80
|
+
<td style="width:100px;">Bit Length</td>
|
81
|
+
<td style="width:200px;">Number Of Replications</td>
|
82
|
+
</tr>
|
83
|
+
<tr>
|
84
|
+
<td>year</td>
|
85
|
+
<td>UInt</td>
|
86
|
+
<td>13</td>
|
87
|
+
<td>1</td>
|
88
|
+
</tr>
|
89
|
+
<tr>
|
90
|
+
<td>month</td>
|
91
|
+
<td>UInt</td>
|
92
|
+
<td>9</td>
|
93
|
+
<td>1</td>
|
94
|
+
</tr>
|
95
|
+
<tr>
|
96
|
+
<td>day</td>
|
97
|
+
<td>UInt</td>
|
98
|
+
<td>9</td>
|
99
|
+
<td>1</td>
|
100
|
+
</tr>
|
101
|
+
</table>
|
102
|
+
|
103
|
+
You can define MyImage structure in ruby program as following code.
|
104
|
+
|
105
|
+
|
106
|
+
require 'binary_parser'
|
107
|
+
|
108
|
+
class MyDate < BinaryParser::TemplateBase
|
109
|
+
require 'date'
|
110
|
+
|
111
|
+
Def do
|
112
|
+
data :year, UInt, 13
|
113
|
+
data :month, UInt, 9
|
114
|
+
data :day, UInt, 9
|
115
|
+
end
|
116
|
+
|
117
|
+
def to_date
|
118
|
+
return Date.new(year.to_i, month.to_i, day.to_i)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
class MyImage < BinaryParser::TemplateBase
|
123
|
+
Def do
|
124
|
+
data :height, UInt, 8
|
125
|
+
data :width, UInt, 8
|
126
|
+
|
127
|
+
TIMES var(:height), :i do
|
128
|
+
TIMES var(:width), :j do
|
129
|
+
data :R, UInt, 8
|
130
|
+
data :G, UInt, 8
|
131
|
+
data :B, UInt, 8
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
data :has_date, Flag, 1
|
136
|
+
IF cond(:has_date){|v| v.flagged?} do
|
137
|
+
data :date, MyDate, 31
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
And then you can parse and read binay-data of MyImage as follows.
|
144
|
+
|
145
|
+
File.open('my_image.bin', 'rb') do |f|
|
146
|
+
image = MyImage.new(f.read)
|
147
|
+
print "Image size: #{image.height.to_i}x#{image.width.to_i}\n"
|
148
|
+
ul = image.i[0].j[0]
|
149
|
+
print "RGB color at the first is (#{ul.R.to_i}, #{ul.G.to_i}, #{ul.B.to_i})\n"
|
150
|
+
print "Image date: #{image.date.to_date}\n"
|
151
|
+
end
|
152
|
+
|
153
|
+
|
154
|
+
If 'my_image.bin' is binary-data-file of [0x02, 0x02, 0xe7,0x39,0x62, 0x00,0x00,0x00, 0xe7,0x39,0x62, 0x00,0x00,0x00, 0x9f, 0x78, 0x08, 0x03],
|
155
|
+
you can get output as follows.
|
156
|
+
|
157
|
+
Image size: 2x2
|
158
|
+
RGB color at the first is (231, 57, 98)
|
159
|
+
Image date: 2014-04-03
|
160
|
+
|
161
|
+
|
162
|
+
For your information, you can dump all binary-data's information as follows.
|
163
|
+
|
164
|
+
File.open('my_image.bin', 'rb') do |f|
|
165
|
+
image = MyImage.new(f.read)
|
166
|
+
image.show(true)
|
167
|
+
end
|
168
|
+
|
169
|
+
|
170
|
+
### Example 2 ###
|
171
|
+
You can also define other structures as follows.
|
172
|
+
|
173
|
+
class DefExample < BinaryParser::TemplateBase
|
174
|
+
Def do
|
175
|
+
data :loop_byte_length, UInt, 8
|
176
|
+
|
177
|
+
# Loop until 'loop_byte_length' * 8 bits are parsed.
|
178
|
+
SPEND var(:loop_byte_length) * 8, :list do
|
179
|
+
data :length, UInt, 8
|
180
|
+
# Specifying length by neigbor value.
|
181
|
+
data :data, Binary, var(:length) * 8
|
182
|
+
end
|
183
|
+
|
184
|
+
data :v1, UInt, 8
|
185
|
+
data :v2, UInt, 8
|
186
|
+
|
187
|
+
# Number of Condition variables is arbitary.
|
188
|
+
IF cond(:v1, :v2){|v1, v2| v1.to_i == v2.to_i} do
|
189
|
+
# +, -, *, / is available for var. (Order of [Integer op Variable] is NG.)
|
190
|
+
data :v3, UInt, (var(:v1) + var(:v2)) * 8
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
Check this definition by giving some binary-data and calling show method as follows.
|
196
|
+
|
197
|
+
i = DefExample.new([0x05, 0x01, 0xff, 0x02, 0xff, 0xff, 0x01, 0x01, 0x01, 0x01].pack("C*"))
|
198
|
+
i.show(true)
|
199
|
+
|
200
|
+
|
201
|
+
### Example 3 ###
|
202
|
+
If you want to operate Stream-data, StreamTemplateBase class is useful. Define stream as follows.
|
203
|
+
|
204
|
+
class StreamExample < BinaryParser::StreamTemplateBase
|
205
|
+
# Stream which consists of every 4 byte binary-data.
|
206
|
+
Def(4) do
|
207
|
+
data :data1, UInt, 8
|
208
|
+
data :data2, Binary, 24
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
And then, get structures from the stream as follows.
|
213
|
+
|
214
|
+
File.open('my_image.bin', 'rb') do |f|
|
215
|
+
stream = StreamExample.new(f)
|
216
|
+
packet = stream.get_next
|
217
|
+
puts "data1: #{packet.data1.to_i}, data2: #{packet.data2.to_s}"
|
218
|
+
stream.get_next.show(true)
|
219
|
+
end
|
220
|
+
|
221
|
+
StreamTemplateBase has many useful method to choose structures from the stream.
|
222
|
+
If you want to know detail of these methods, please read documentation or concerned source-files.
|
223
|
+
|
224
|
+
|
225
|
+
Documentation
|
226
|
+
--------------
|
227
|
+
I'm sorry, but only RDoc (auto-generated documentation) is now available.
|
228
|
+
For example, you can read RDoc on web browser by following operations.
|
229
|
+
|
230
|
+
$ gem install binary_parser
|
231
|
+
$ gem server
|
232
|
+
Server started at http://0.0.0.0:8808
|
233
|
+
|
234
|
+
Access shown address by web browser.
|
235
|
+
|
236
|
+
|
237
|
+
Versions
|
238
|
+
--------
|
239
|
+
1.0.0 April 6, 2014
|
data/Rakefile
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# -*- mode: ruby; coding: utf-8 -*-
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'binary_parser/version'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = "binary_parser"
|
9
|
+
spec.version = BinaryParser::VERSION
|
10
|
+
spec.authors = ["rokugats(u)"]
|
11
|
+
spec.email = ["sasasawada@gmail.com"]
|
12
|
+
spec.summary = "An elegant DSL library for parsing binary-data."
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
|
24
|
+
spec.description = <<-END
|
25
|
+
This library can parse all kind of binary-data structures including non-fixed length of structures and nested structures.
|
26
|
+
For generic parsing, loop and condition(if) statement to define structures is provided in this library.
|
27
|
+
Of course, values of neighbor binary-data can be used as the other binary-data's specification of length.
|
28
|
+
|
29
|
+
Furthermore, this library handles all binary-data under the lazy evaluation.
|
30
|
+
So you can read required parts of a binary-data very quickly even if whole of the binary-data is too big,
|
31
|
+
END
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require "binary_parser/version"
|
2
|
+
|
3
|
+
module BinaryParser
|
4
|
+
|
5
|
+
LIBRARY_ROOT_PATH = File.dirname(File.expand_path(File.dirname(__FILE__)))
|
6
|
+
|
7
|
+
# load general class file
|
8
|
+
GENERAL_CLASS_DIR = '/lib/general_class/'
|
9
|
+
GENERAL_CLASS_FILES =
|
10
|
+
['abstract_binary',
|
11
|
+
'expression.rb',
|
12
|
+
'bit_position.rb',
|
13
|
+
'condition.rb'
|
14
|
+
]
|
15
|
+
|
16
|
+
GENERAL_CLASS_FILES.each do |path|
|
17
|
+
require LIBRARY_ROOT_PATH + GENERAL_CLASS_DIR + path
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
# load built-in template file
|
22
|
+
class TemplateBase; end
|
23
|
+
BUILT_IN_TEMPLATE_DIR = '/lib/built_in_template/'
|
24
|
+
BUILT_IN_TEMPLATE_FILES =
|
25
|
+
['uint.rb',
|
26
|
+
'flag.rb',
|
27
|
+
'binary.rb'
|
28
|
+
]
|
29
|
+
|
30
|
+
BUILT_IN_TEMPLATE_FILES.each do |path|
|
31
|
+
require LIBRARY_ROOT_PATH + BUILT_IN_TEMPLATE_DIR + path
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
# load library main file
|
36
|
+
LIB_DIR = '/lib/'
|
37
|
+
LIB_FILES =
|
38
|
+
['loop_list.rb',
|
39
|
+
'scope.rb',
|
40
|
+
'structure_definition.rb',
|
41
|
+
'template_base.rb',
|
42
|
+
'stream_template_base.rb',
|
43
|
+
'nameless_template.rb',
|
44
|
+
'error.rb'
|
45
|
+
]
|
46
|
+
|
47
|
+
LIB_FILES.each do |path|
|
48
|
+
require LIBRARY_ROOT_PATH + LIB_DIR + path
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module BinaryParser
|
2
|
+
module BuiltInTemplate
|
3
|
+
class Binary < TemplateBase
|
4
|
+
def content_description
|
5
|
+
chars = to_chars
|
6
|
+
bytes = chars[0, 5].map{|i| sprintf("0x%02x", i)}.join(", ")
|
7
|
+
return "[" + bytes + (chars.length > 5 ? ", ..." : "") + "]"
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module BinaryParser
|
2
|
+
module BuiltInTemplate
|
3
|
+
class Flag < TemplateBase
|
4
|
+
|
5
|
+
def on?
|
6
|
+
return to_i[0] == 1
|
7
|
+
end
|
8
|
+
|
9
|
+
alias_method :flagged?, :on?
|
10
|
+
|
11
|
+
def off?
|
12
|
+
return !on?
|
13
|
+
end
|
14
|
+
|
15
|
+
alias_method :unflagged?, :off?
|
16
|
+
|
17
|
+
def content_description
|
18
|
+
on? ? "true" : "false"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/error.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
module BinaryParser
|
2
|
+
|
3
|
+
# User do Invalid Parsing Definition.
|
4
|
+
DefinitionError = Class.new(StandardError)
|
5
|
+
|
6
|
+
# User do Bad Manipulation.
|
7
|
+
BadManipulationError = Class.new(StandardError)
|
8
|
+
|
9
|
+
# Undefined Data is referenced.
|
10
|
+
UndefinedError = Class.new(StandardError)
|
11
|
+
|
12
|
+
# Invalid Binary Pattern is parsed.
|
13
|
+
ParsingError = Class.new(StandardError)
|
14
|
+
|
15
|
+
# Invalid Binary Manipulation is done.
|
16
|
+
BadBinaryManipulationError = Class.new(StandardError)
|
17
|
+
|
18
|
+
# Assertion Error.
|
19
|
+
# If this error occurs in regular use, probably this library(binary_parser) has Bugs.
|
20
|
+
ProgramAssertionError = Class.new(StandardError)
|
21
|
+
end
|