sql-server-raw-file 0.0.1 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/README.md +3 -1
- data/fixtures/current_sql_index +1 -1
- data/lib/sql_server_dts_ns.rb +3 -1
- data/lib/sql_server_raw_file.rb +46 -30
- data/sql-server-raw-file.gemspec +4 -1
- data/test/test_sql_data_and_header.rb +33 -14
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 41f203edbad49fe5604df8564fe39edc4f70aef3
|
4
|
+
data.tar.gz: 6a9215a1fd7c64097ec3e50c3d08649aa76db7d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a3d9a2c0760386dc63e4286b8a77b4830da2bcaaa0d303f0e7594cf810cde1409e8fb622e5208992832c5f71623a38cef9761bc7115257a6fd546520bb68068
|
7
|
+
data.tar.gz: d4bbae1b9efd3934109b051b87893ef4a7d4cb1cebf84e78cedd5dab5d2454765bddfd41d10c498e518dac7b76f2669d901b442f5b72b665acc76090199bc009
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## 0.1.1 (Jan 20, 2016)
|
2
|
+
|
3
|
+
Features:
|
4
|
+
|
5
|
+
- added SqlServerDts::RawFile#eof?
|
6
|
+
* check io-stream eof
|
7
|
+
- added SqlServerDts::RawFile#pos
|
8
|
+
* get current position in inner io-stream object
|
9
|
+
|
10
|
+
Bugfixes:
|
11
|
+
|
12
|
+
- missing method SqlServerDts::RawFile#close
|
13
|
+
* is need to close inner io-stream object in some cases
|
14
|
+
|
1
15
|
## 0.0.1 (Jan 5, 2016)
|
2
16
|
|
3
17
|
Features:
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# sql-server-raw-file: sql-server-raw-file lib
|
2
2
|
|
3
|
-
Converts SQL Server
|
3
|
+
Converts SQL Server Integration Services Raw File source to hash.
|
4
4
|
|
5
5
|
### Installation and usage
|
6
6
|
|
@@ -42,6 +42,8 @@ SqlServerDts::RawFile.new('/path/to/sql/raw/file') { |raw_obj, raw_io|
|
|
42
42
|
# ...
|
43
43
|
}
|
44
44
|
# end example 2
|
45
|
+
|
46
|
+
3) more detailed examples in ./test/test_sql_data_and_header.rb
|
45
47
|
```
|
46
48
|
|
47
49
|
### Troubleshooting
|
data/fixtures/current_sql_index
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1
|
data/lib/sql_server_dts_ns.rb
CHANGED
data/lib/sql_server_raw_file.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# coding: utf-8
|
2
|
-
|
2
|
+
|
3
|
+
require(File.expand_path('../sql_server_dts_ns.rb',__FILE__))
|
3
4
|
|
4
5
|
# SqlServerDts::RawFile class
|
5
6
|
class SqlServerDts::RawFile
|
@@ -38,19 +39,19 @@ class SqlServerDts::RawFile
|
|
38
39
|
# in other side returns self
|
39
40
|
def initialize(io,init_pos=nil)
|
40
41
|
if io.is_a?(IO)
|
41
|
-
@
|
42
|
-
@
|
42
|
+
@raw_io = io
|
43
|
+
@raw_io.binmode unless @raw_io.binmode?
|
43
44
|
read_header
|
44
|
-
@
|
45
|
+
@raw_io.seek(init_pos || @data_pos)
|
45
46
|
else
|
46
|
-
@
|
47
|
+
@raw_io = File.new(io,'rb')
|
47
48
|
read_header
|
48
|
-
@
|
49
|
+
@raw_io.seek(init_pos || @data_pos)
|
49
50
|
if block_given?
|
50
51
|
begin
|
51
|
-
yield [self,@
|
52
|
+
yield [self,@raw_io]
|
52
53
|
ensure
|
53
|
-
@
|
54
|
+
@raw_io.close
|
54
55
|
end
|
55
56
|
else
|
56
57
|
self
|
@@ -60,21 +61,21 @@ class SqlServerDts::RawFile
|
|
60
61
|
|
61
62
|
# read metadata (header) raw file and fill inner array of metadata
|
62
63
|
def read_header
|
63
|
-
raise IOError, 'Stream is closed' if @
|
64
|
+
raise IOError, 'Stream is closed' if @raw_io.closed?
|
64
65
|
begin
|
65
|
-
@
|
66
|
-
@version = @
|
66
|
+
@raw_io.rewind
|
67
|
+
@version = @raw_io.read(4).unpack('H2H2H2H2').join('.')
|
67
68
|
raise UnsupportedRawVersionError, "Version #{@version} unsupported " unless SUPPORTED_RAW_VERSIONS.include?(@version)
|
68
69
|
@fields_info = []
|
69
|
-
fields_count = @
|
70
|
+
fields_count = @raw_io.read(4).unpack('I')[0]
|
70
71
|
raise FieldsNotExistsError, 'Fields not exists (fields_count=0)' if fields_count==0
|
71
72
|
@nil_mask_len = fields_count/8+(fields_count-fields_count/8*8>0?1:0)
|
72
73
|
fields_count.times {
|
73
|
-
name_len = @
|
74
|
-
field_name = @
|
74
|
+
name_len = @raw_io.read(4).unpack('I')[0]
|
75
|
+
field_name = @raw_io.read(name_len*2).encode(DEFAULT_ENCODING,DEFAULT_RAW_FILE_ENCODING,{ invalid: :replace, undef: :replace })
|
75
76
|
(@version == '00.10.01.00') ?
|
76
|
-
values_arr = @
|
77
|
-
values_arr = @
|
77
|
+
values_arr = @raw_io.read(40).unpack('I4iI5') :
|
78
|
+
values_arr = @raw_io.read(32).unpack('I4iI3')
|
78
79
|
field_info = {
|
79
80
|
name: field_name,
|
80
81
|
data_type: values_arr[0],
|
@@ -93,7 +94,7 @@ class SqlServerDts::RawFile
|
|
93
94
|
rescue
|
94
95
|
raise
|
95
96
|
else
|
96
|
-
@data_pos = @
|
97
|
+
@data_pos = @raw_io.pos
|
97
98
|
end
|
98
99
|
end
|
99
100
|
|
@@ -101,9 +102,9 @@ class SqlServerDts::RawFile
|
|
101
102
|
#
|
102
103
|
# @return [Hash] Hash of converted raw data (one line)
|
103
104
|
def read_line
|
104
|
-
raise IOError, 'Stream is closed' if @
|
105
|
+
raise IOError, 'Stream is closed' if @raw_io.closed?
|
105
106
|
begin
|
106
|
-
nil_mask = @
|
107
|
+
nil_mask = @raw_io.read(@nil_mask_len).unpack('b*')[0].each_char.to_a
|
107
108
|
data_row = []
|
108
109
|
@fields_info.each_with_index { |field_info, index|
|
109
110
|
field_value =
|
@@ -112,28 +113,28 @@ class SqlServerDts::RawFile
|
|
112
113
|
else
|
113
114
|
case field_info[:data_type]
|
114
115
|
when 3 # int, NULL w/o cast
|
115
|
-
@
|
116
|
+
@raw_io.read(field_info[:max_length]).unpack('I')[0]
|
116
117
|
when 5 # float
|
117
|
-
@
|
118
|
+
@raw_io.read(field_info[:max_length]).unpack('D')[0]
|
118
119
|
when 11 # bit
|
119
|
-
(@
|
120
|
+
(@raw_io.read(field_info[:max_length]).unpack('B')[0] == '1') ? true : false
|
120
121
|
when 20 # bigint
|
121
|
-
@
|
122
|
+
@raw_io.read(field_info[:max_length]).unpack('L')[0]
|
122
123
|
when 129 # varchar, char
|
123
|
-
data_len = @
|
124
|
-
@
|
124
|
+
data_len = @raw_io.read(4).unpack('I')[0]
|
125
|
+
@raw_io.read(data_len).encode({invalid: :replace, undef: :replace})
|
125
126
|
when 130 # nvarchar, nchar
|
126
|
-
data_len = @
|
127
|
-
@
|
127
|
+
data_len = @raw_io.read(4).unpack('I')[0]
|
128
|
+
@raw_io.read(data_len*2).encode(DEFAULT_ENCODING, DEFAULT_RAW_FILE_ENCODING, {invalid: :replace, undef: :replace})
|
128
129
|
when 131 # numeric, decimal
|
129
130
|
# precision, scale, sign, value
|
130
|
-
values_arr = @
|
131
|
+
values_arr = @raw_io.read(field_info[:max_length]).unpack('C3Q')
|
131
132
|
"#{('-' if values_arr[2] == 0)}#{values_arr[3].to_s.insert((values_arr[3].to_s.length-values_arr[1]), '.')}"
|
132
133
|
when 135 # datetime
|
133
|
-
values_arr = @
|
134
|
+
values_arr = @raw_io.read(field_info[:max_length]).unpack('S6I')
|
134
135
|
Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime('%F %T.%L')
|
135
136
|
when 304 # datetime2
|
136
|
-
values_arr = @
|
137
|
+
values_arr = @raw_io.read(field_info[:max_length]).unpack('S6I')
|
137
138
|
Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime("%F %T.%#{field_info[:scale]}N")
|
138
139
|
else
|
139
140
|
raise UnknownDataTypeError, "Unknown data type #{field_info[:data_type]}"
|
@@ -159,4 +160,19 @@ class SqlServerDts::RawFile
|
|
159
160
|
def version
|
160
161
|
@version
|
161
162
|
end
|
163
|
+
|
164
|
+
# close io-stream
|
165
|
+
def close
|
166
|
+
@raw_io.close unless @raw_io.closed?
|
167
|
+
end
|
168
|
+
|
169
|
+
# check io-stream eof
|
170
|
+
def eof?
|
171
|
+
@raw_io.eof?
|
172
|
+
end
|
173
|
+
|
174
|
+
# get current position in inner io-stream object
|
175
|
+
def pos
|
176
|
+
@raw_io.pos
|
177
|
+
end
|
162
178
|
end
|
data/sql-server-raw-file.gemspec
CHANGED
@@ -1,6 +1,9 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
1
3
|
require 'rubygems'
|
2
4
|
|
3
|
-
require "#{File.dirname(__FILE__)}/lib/sql_server_dts_ns"
|
5
|
+
#require "#{File.dirname(__FILE__)}/lib/sql_server_dts_ns"
|
6
|
+
require(File.expand_path('../lib/sql_server_dts_ns.rb',__FILE__))
|
4
7
|
|
5
8
|
Gem::Specification.new do |s|
|
6
9
|
s.name = 'sql-server-raw-file'
|
@@ -1,30 +1,49 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
|
2
3
|
require 'csv'
|
3
4
|
require 'json'
|
4
5
|
|
5
|
-
require
|
6
|
+
require(File.expand_path('../../lib/sql_server_raw_file.rb',__FILE__))
|
6
7
|
|
7
|
-
fixtures_path =
|
8
|
-
|
8
|
+
fixtures_path = File.expand_path('../../fixtures',__FILE__)+'/'
|
9
|
+
raw_file_name = %w(sql09.raw sql10.raw sql11.raw)
|
9
10
|
|
10
11
|
# current_fixture_index = 0
|
11
12
|
current_fixture_index = File.read(fixtures_path+'/current_sql_index').to_i
|
13
|
+
raw_path = fixtures_path+raw_file_name[current_fixture_index]
|
12
14
|
|
13
|
-
# show
|
14
|
-
|
15
|
+
# show version of raw file
|
16
|
+
# signature: new(io)
|
17
|
+
# @param io [IO] IO-stream
|
18
|
+
# @return [SqlServerDts::RawFile]
|
19
|
+
File.open(raw_path) { |raw_io|
|
15
20
|
raw_obj = SqlServerDts::RawFile.new(raw_io)
|
16
21
|
puts "RAW VERSION: #{raw_obj.version}"
|
17
|
-
puts '=== BEGIN HEADER ==='
|
18
|
-
puts raw_obj.header
|
19
|
-
File.open("#{fixtures_path+File.basename(raw_io.path, '.*')}_header.json",'wt') { |json_io|
|
20
|
-
header_json = JSON.pretty_generate(raw_obj.header)
|
21
|
-
json_io.write(header_json)
|
22
|
-
}
|
23
|
-
puts '=== END HEADER ==='
|
24
22
|
}
|
25
23
|
|
26
|
-
# show
|
27
|
-
|
24
|
+
# show header of raw file and convert it into hash and save it as json-file
|
25
|
+
# signature: new(path)
|
26
|
+
# @param path [String] Path to raw file
|
27
|
+
# @return [SqlServerDts::RawFile]
|
28
|
+
raw_obj = SqlServerDts::RawFile.new(raw_path)
|
29
|
+
puts '=== BEGIN HEADER ==='
|
30
|
+
puts raw_obj.header
|
31
|
+
File.open("#{fixtures_path+File.basename(raw_path, '.*')}_header.json",'wt') { |json_io|
|
32
|
+
header_json = JSON.pretty_generate(raw_obj.header)
|
33
|
+
json_io.write(header_json)
|
34
|
+
}
|
35
|
+
puts '=== END HEADER ==='
|
36
|
+
# get current position into inner IO-stream object for further using
|
37
|
+
init_pos = raw_obj.pos
|
38
|
+
# need to close inner IO-stream object
|
39
|
+
raw_obj.close
|
40
|
+
|
41
|
+
# show data of raw file in rows table and convert it into hash and save it as csv-file
|
42
|
+
# signature: new(path,init_pos)
|
43
|
+
# @param path [String] Path to raw file
|
44
|
+
# @param init_pos [String] Init pos in raw file where data rows from
|
45
|
+
# @return [ [SqlServerDts::RawFile,IO] ] block_given? == true
|
46
|
+
SqlServerDts::RawFile.new(raw_path,init_pos) { |raw_obj, raw_io|
|
28
47
|
puts '=== BEGIN DATA ==='
|
29
48
|
i=0
|
30
49
|
fields_arr = []
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sql-server-raw-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dmitriy Mullo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Converts SQL Server Integration Services Raw File source to hash.
|
14
14
|
email:
|