sql-server-raw-file 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +20 -0
- data/LICENSE.md +22 -0
- data/README.md +50 -0
- data/fixtures/current_sql_index +1 -0
- data/fixtures/select-fixture-data-SQL_Server_90.sql +3 -0
- data/fixtures/select-fixture-data.sql +3 -0
- data/fixtures/sql09.raw +0 -0
- data/fixtures/sql09_data.csv +3 -0
- data/fixtures/sql09_header.json +171 -0
- data/fixtures/sql10.raw +0 -0
- data/fixtures/sql10_data.csv +3 -0
- data/fixtures/sql10_header.json +171 -0
- data/fixtures/sql11.raw +0 -0
- data/fixtures/sql11_data.csv +3 -0
- data/fixtures/sql11_header.json +171 -0
- data/lib/sql_server_dts_ns.rb +5 -0
- data/lib/sql_server_raw_file.rb +162 -0
- data/sql-server-raw-file.gemspec +17 -0
- data/test/test_sql_data_and_header.rb +44 -0
- metadata +63 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ea75b306d331a3e23df7a321c868778e60266a13
|
4
|
+
data.tar.gz: b79c539f190cc42633fbca1dd980a7a878e60b88
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ad451f419c448ca2bc864875c4141c6931457e0bdf73113f1e5fb9ec170e2f30a882c9358dc9e0868d19bca2df7e627ae3bcbb36838eb746b324aef7ef36fe39
|
7
|
+
data.tar.gz: f415ea6b1439db87d8a930dec480d5eb33bb2fe74d1f982e79944af80dd1abad365edc34025b2bad2cef3fd6651096a119ee3bd027b4f290334a5b7fc1a67cb7
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
## 0.0.1 (Jan 5, 2016)
|
2
|
+
|
3
|
+
Features:
|
4
|
+
|
5
|
+
- added support of SQL Server Dts Raw File versions:
|
6
|
+
* 00.90.00.00 => 9 (SQL Server: 9/10)
|
7
|
+
* 00.10.01.00 => 10 (SQL Server: 11)
|
8
|
+
- added support of basic data types:
|
9
|
+
* int (i.e. NULL w/o cast)
|
10
|
+
* float
|
11
|
+
* bit
|
12
|
+
* bigint
|
13
|
+
* (n)(var)char
|
14
|
+
* decimal, numeric
|
15
|
+
* datetime
|
16
|
+
* datetime2 (only SQL Server 10/11)
|
17
|
+
|
18
|
+
Bugfixes:
|
19
|
+
|
20
|
+
- initial release
|
data/LICENSE.md
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2016 Dmitriy Mullo
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# sql-server-raw-file: sql-server-raw-file lib
|
2
|
+
|
3
|
+
Converts SQL Server Dts Raw File source to hash.
|
4
|
+
|
5
|
+
### Installation and usage
|
6
|
+
|
7
|
+
```
|
8
|
+
1) gem install sql-server-raw-file
|
9
|
+
2) usage (examples):
|
10
|
+
|
11
|
+
# begin example 1
|
12
|
+
require 'sql_server_raw_file'
|
13
|
+
|
14
|
+
File.open('/path/to/sql/raw/file') { |raw_io|
|
15
|
+
raw_obj = SqlServerDts::RawFile.new(raw_io)
|
16
|
+
# ...
|
17
|
+
puts raw_obj.version
|
18
|
+
puts raw_obj.header
|
19
|
+
# ...
|
20
|
+
until raw_io.eof?
|
21
|
+
# ...
|
22
|
+
puts raw_obj.read_line
|
23
|
+
# ...
|
24
|
+
end
|
25
|
+
# ...
|
26
|
+
}
|
27
|
+
# end example 1
|
28
|
+
|
29
|
+
# begin example 2
|
30
|
+
require 'sql_server_raw_file'
|
31
|
+
|
32
|
+
SqlServerDts::RawFile.new('/path/to/sql/raw/file') { |raw_obj, raw_io|
|
33
|
+
# ...
|
34
|
+
puts raw_obj.version
|
35
|
+
puts raw_obj.header
|
36
|
+
# ...
|
37
|
+
until raw_io.eof?
|
38
|
+
# ...
|
39
|
+
puts raw_obj.read_line
|
40
|
+
# ...
|
41
|
+
end
|
42
|
+
# ...
|
43
|
+
}
|
44
|
+
# end example 2
|
45
|
+
```
|
46
|
+
|
47
|
+
### Troubleshooting
|
48
|
+
|
49
|
+
mailto:d.a.mullo1981@gmail.com
|
50
|
+
subject: sql-server-raw-file issue
|
@@ -0,0 +1 @@
|
|
1
|
+
2
|
@@ -0,0 +1,3 @@
|
|
1
|
+
select cast(1 as int) [int], cast(3.14159265 as float) [float], cast(1 as bit) [bit], cast(POWER(2,30) as bigint) [bigint], cast('This is a variable length char string' as varchar(128)) [varchar], cast('This is a char string' as char(128)) [char], cast('��� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ������ � �������' as nchar(128)) [nchar], cast(9.11 as decimal(18,6)) [decimal], cast(7.62 as numeric(18,6)) [numeric], getdate() [datetime], NULL [datetime2], NULL AS [NULL]
|
2
|
+
UNION ALL
|
3
|
+
select cast(2 as int) [int], cast(-37.0/3.0 as float) [float], cast(0 as bit) [bit], cast(-POWER(3,19) as bigint) [bigint], cast('This is another variable length char string' as varchar(128)) [varchar], cast('This is another char string' as char(128)) [char], cast('��� ��� ���� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ��� ���� ������ � �������' as nchar(128)) [nchar], cast(-99.999 as decimal(18,6)) [decimal], cast(-5.56 as numeric(18,6)) [numeric], getdate() [datetime], NULL [datetime2], NULL AS [NULL]
|
@@ -0,0 +1,3 @@
|
|
1
|
+
select cast(1 as int) [int], cast(3.14159265 as float) [float], cast(1 as bit) [bit], cast(POWER(2,30) as bigint) [bigint], cast('This is a variable length char string' as varchar(128)) [varchar], cast('This is a char string' as char(128)) [char], cast('��� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ������ � �������' as nchar(128)) [nchar], cast(9.11 as decimal(18,6)) [decimal], cast(7.62 as numeric(18,6)) [numeric], getdate() [datetime], CAST(GETDATE() AS DATETIME2(0)) [datetime2], NULL AS [NULL]
|
2
|
+
UNION ALL
|
3
|
+
select cast(2 as int) [int], cast(-37.0/3.0 as float) [float], cast(0 as bit) [bit], cast(-POWER(3,19) as bigint) [bigint], cast('This is another variable length char string' as varchar(128)) [varchar], cast('This is another char string' as char(128)) [char], cast('��� ��� ���� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ��� ���� ������ � �������' as nchar(128)) [nchar], cast(-99.999 as decimal(18,6)) [decimal], cast(-5.56 as numeric(18,6)) [numeric], getdate() [datetime], CAST(GETDATE() AS DATETIME2(7)) [datetime2], NULL AS [NULL]
|
data/fixtures/sql09.raw
ADDED
Binary file
|
@@ -0,0 +1,3 @@
|
|
1
|
+
int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
|
2
|
+
1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 09:26:52.747
|
3
|
+
2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 09:26:52.747
|
@@ -0,0 +1,171 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"name": "int",
|
4
|
+
"data_type": 3,
|
5
|
+
"max_length": 4,
|
6
|
+
"p2": 52,
|
7
|
+
"p3": 0,
|
8
|
+
"p4": -1,
|
9
|
+
"precision": 0,
|
10
|
+
"scale": 0,
|
11
|
+
"code_page": 0,
|
12
|
+
"p8": null,
|
13
|
+
"p9": null
|
14
|
+
},
|
15
|
+
{
|
16
|
+
"name": "float",
|
17
|
+
"data_type": 5,
|
18
|
+
"max_length": 8,
|
19
|
+
"p2": 56,
|
20
|
+
"p3": 4,
|
21
|
+
"p4": -1,
|
22
|
+
"precision": 0,
|
23
|
+
"scale": 0,
|
24
|
+
"code_page": 0,
|
25
|
+
"p8": null,
|
26
|
+
"p9": null
|
27
|
+
},
|
28
|
+
{
|
29
|
+
"name": "bit",
|
30
|
+
"data_type": 11,
|
31
|
+
"max_length": 2,
|
32
|
+
"p2": 392,
|
33
|
+
"p3": 8,
|
34
|
+
"p4": -1,
|
35
|
+
"precision": 0,
|
36
|
+
"scale": 0,
|
37
|
+
"code_page": 0,
|
38
|
+
"p8": null,
|
39
|
+
"p9": null
|
40
|
+
},
|
41
|
+
{
|
42
|
+
"name": "bigint",
|
43
|
+
"data_type": 20,
|
44
|
+
"max_length": 8,
|
45
|
+
"p2": 80,
|
46
|
+
"p3": 12,
|
47
|
+
"p4": -1,
|
48
|
+
"precision": 0,
|
49
|
+
"scale": 0,
|
50
|
+
"code_page": 0,
|
51
|
+
"p8": null,
|
52
|
+
"p9": null
|
53
|
+
},
|
54
|
+
{
|
55
|
+
"name": "varchar",
|
56
|
+
"data_type": 129,
|
57
|
+
"max_length": 128,
|
58
|
+
"p2": 92,
|
59
|
+
"p3": 16,
|
60
|
+
"p4": -1,
|
61
|
+
"precision": 0,
|
62
|
+
"scale": 0,
|
63
|
+
"code_page": 1251,
|
64
|
+
"p8": null,
|
65
|
+
"p9": null
|
66
|
+
},
|
67
|
+
{
|
68
|
+
"name": "char",
|
69
|
+
"data_type": 129,
|
70
|
+
"max_length": 128,
|
71
|
+
"p2": 244,
|
72
|
+
"p3": 20,
|
73
|
+
"p4": -1,
|
74
|
+
"precision": 0,
|
75
|
+
"scale": 0,
|
76
|
+
"code_page": 1251,
|
77
|
+
"p8": null,
|
78
|
+
"p9": null
|
79
|
+
},
|
80
|
+
{
|
81
|
+
"name": "nvarchar",
|
82
|
+
"data_type": 130,
|
83
|
+
"max_length": 128,
|
84
|
+
"p2": 396,
|
85
|
+
"p3": 24,
|
86
|
+
"p4": -1,
|
87
|
+
"precision": 0,
|
88
|
+
"scale": 0,
|
89
|
+
"code_page": 0,
|
90
|
+
"p8": null,
|
91
|
+
"p9": null
|
92
|
+
},
|
93
|
+
{
|
94
|
+
"name": "nchar",
|
95
|
+
"data_type": 130,
|
96
|
+
"max_length": 128,
|
97
|
+
"p2": 656,
|
98
|
+
"p3": 28,
|
99
|
+
"p4": -1,
|
100
|
+
"precision": 0,
|
101
|
+
"scale": 0,
|
102
|
+
"code_page": 0,
|
103
|
+
"p8": null,
|
104
|
+
"p9": null
|
105
|
+
},
|
106
|
+
{
|
107
|
+
"name": "decimal",
|
108
|
+
"data_type": 131,
|
109
|
+
"max_length": 19,
|
110
|
+
"p2": 221,
|
111
|
+
"p3": 32,
|
112
|
+
"p4": -1,
|
113
|
+
"precision": 18,
|
114
|
+
"scale": 6,
|
115
|
+
"code_page": 0,
|
116
|
+
"p8": null,
|
117
|
+
"p9": null
|
118
|
+
},
|
119
|
+
{
|
120
|
+
"name": "numeric",
|
121
|
+
"data_type": 131,
|
122
|
+
"max_length": 19,
|
123
|
+
"p2": 373,
|
124
|
+
"p3": 36,
|
125
|
+
"p4": -1,
|
126
|
+
"precision": 18,
|
127
|
+
"scale": 6,
|
128
|
+
"code_page": 0,
|
129
|
+
"p8": null,
|
130
|
+
"p9": null
|
131
|
+
},
|
132
|
+
{
|
133
|
+
"name": "datetime",
|
134
|
+
"data_type": 135,
|
135
|
+
"max_length": 16,
|
136
|
+
"p2": 64,
|
137
|
+
"p3": 40,
|
138
|
+
"p4": -1,
|
139
|
+
"precision": 0,
|
140
|
+
"scale": 0,
|
141
|
+
"code_page": 0,
|
142
|
+
"p8": null,
|
143
|
+
"p9": null
|
144
|
+
},
|
145
|
+
{
|
146
|
+
"name": "datetime2",
|
147
|
+
"data_type": 3,
|
148
|
+
"max_length": 4,
|
149
|
+
"p2": 88,
|
150
|
+
"p3": 44,
|
151
|
+
"p4": -1,
|
152
|
+
"precision": 0,
|
153
|
+
"scale": 0,
|
154
|
+
"code_page": 0,
|
155
|
+
"p8": null,
|
156
|
+
"p9": null
|
157
|
+
},
|
158
|
+
{
|
159
|
+
"name": "NULL",
|
160
|
+
"data_type": 3,
|
161
|
+
"max_length": 4,
|
162
|
+
"p2": 240,
|
163
|
+
"p3": 48,
|
164
|
+
"p4": -1,
|
165
|
+
"precision": 0,
|
166
|
+
"scale": 0,
|
167
|
+
"code_page": 0,
|
168
|
+
"p8": null,
|
169
|
+
"p9": null
|
170
|
+
}
|
171
|
+
]
|
data/fixtures/sql10.raw
ADDED
Binary file
|
@@ -0,0 +1,3 @@
|
|
1
|
+
int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
|
2
|
+
1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 08:20:06.823 2016-01-12 08:20:07.0000000
|
3
|
+
2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 08:20:06.823 2016-01-12 08:20:06.8230000
|
@@ -0,0 +1,171 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"name": "int",
|
4
|
+
"data_type": 3,
|
5
|
+
"max_length": 4,
|
6
|
+
"p2": 52,
|
7
|
+
"p3": 0,
|
8
|
+
"p4": -1,
|
9
|
+
"precision": 0,
|
10
|
+
"scale": 0,
|
11
|
+
"code_page": 0,
|
12
|
+
"p8": null,
|
13
|
+
"p9": null
|
14
|
+
},
|
15
|
+
{
|
16
|
+
"name": "float",
|
17
|
+
"data_type": 5,
|
18
|
+
"max_length": 8,
|
19
|
+
"p2": 56,
|
20
|
+
"p3": 4,
|
21
|
+
"p4": -1,
|
22
|
+
"precision": 0,
|
23
|
+
"scale": 0,
|
24
|
+
"code_page": 0,
|
25
|
+
"p8": null,
|
26
|
+
"p9": null
|
27
|
+
},
|
28
|
+
{
|
29
|
+
"name": "bit",
|
30
|
+
"data_type": 11,
|
31
|
+
"max_length": 2,
|
32
|
+
"p2": 256,
|
33
|
+
"p3": 8,
|
34
|
+
"p4": -1,
|
35
|
+
"precision": 0,
|
36
|
+
"scale": 0,
|
37
|
+
"code_page": 0,
|
38
|
+
"p8": null,
|
39
|
+
"p9": null
|
40
|
+
},
|
41
|
+
{
|
42
|
+
"name": "bigint",
|
43
|
+
"data_type": 20,
|
44
|
+
"max_length": 8,
|
45
|
+
"p2": 96,
|
46
|
+
"p3": 12,
|
47
|
+
"p4": -1,
|
48
|
+
"precision": 0,
|
49
|
+
"scale": 0,
|
50
|
+
"code_page": 0,
|
51
|
+
"p8": null,
|
52
|
+
"p9": null
|
53
|
+
},
|
54
|
+
{
|
55
|
+
"name": "varchar",
|
56
|
+
"data_type": 129,
|
57
|
+
"max_length": 128,
|
58
|
+
"p2": 108,
|
59
|
+
"p3": 16,
|
60
|
+
"p4": -1,
|
61
|
+
"precision": 0,
|
62
|
+
"scale": 0,
|
63
|
+
"code_page": 1251,
|
64
|
+
"p8": null,
|
65
|
+
"p9": null
|
66
|
+
},
|
67
|
+
{
|
68
|
+
"name": "char",
|
69
|
+
"data_type": 129,
|
70
|
+
"max_length": 128,
|
71
|
+
"p2": 280,
|
72
|
+
"p3": 20,
|
73
|
+
"p4": -1,
|
74
|
+
"precision": 0,
|
75
|
+
"scale": 0,
|
76
|
+
"code_page": 1251,
|
77
|
+
"p8": null,
|
78
|
+
"p9": null
|
79
|
+
},
|
80
|
+
{
|
81
|
+
"name": "nvarchar",
|
82
|
+
"data_type": 130,
|
83
|
+
"max_length": 128,
|
84
|
+
"p2": 412,
|
85
|
+
"p3": 24,
|
86
|
+
"p4": -1,
|
87
|
+
"precision": 0,
|
88
|
+
"scale": 0,
|
89
|
+
"code_page": 0,
|
90
|
+
"p8": null,
|
91
|
+
"p9": null
|
92
|
+
},
|
93
|
+
{
|
94
|
+
"name": "nchar",
|
95
|
+
"data_type": 130,
|
96
|
+
"max_length": 128,
|
97
|
+
"p2": 672,
|
98
|
+
"p3": 28,
|
99
|
+
"p4": -1,
|
100
|
+
"precision": 0,
|
101
|
+
"scale": 0,
|
102
|
+
"code_page": 0,
|
103
|
+
"p8": null,
|
104
|
+
"p9": null
|
105
|
+
},
|
106
|
+
{
|
107
|
+
"name": "decimal",
|
108
|
+
"data_type": 131,
|
109
|
+
"max_length": 19,
|
110
|
+
"p2": 237,
|
111
|
+
"p3": 32,
|
112
|
+
"p4": -1,
|
113
|
+
"precision": 18,
|
114
|
+
"scale": 6,
|
115
|
+
"code_page": 0,
|
116
|
+
"p8": null,
|
117
|
+
"p9": null
|
118
|
+
},
|
119
|
+
{
|
120
|
+
"name": "numeric",
|
121
|
+
"data_type": 131,
|
122
|
+
"max_length": 19,
|
123
|
+
"p2": 258,
|
124
|
+
"p3": 36,
|
125
|
+
"p4": -1,
|
126
|
+
"precision": 18,
|
127
|
+
"scale": 6,
|
128
|
+
"code_page": 0,
|
129
|
+
"p8": null,
|
130
|
+
"p9": null
|
131
|
+
},
|
132
|
+
{
|
133
|
+
"name": "datetime",
|
134
|
+
"data_type": 135,
|
135
|
+
"max_length": 16,
|
136
|
+
"p2": 64,
|
137
|
+
"p3": 40,
|
138
|
+
"p4": -1,
|
139
|
+
"precision": 0,
|
140
|
+
"scale": 0,
|
141
|
+
"code_page": 0,
|
142
|
+
"p8": null,
|
143
|
+
"p9": null
|
144
|
+
},
|
145
|
+
{
|
146
|
+
"name": "datetime2",
|
147
|
+
"data_type": 304,
|
148
|
+
"max_length": 16,
|
149
|
+
"p2": 80,
|
150
|
+
"p3": 44,
|
151
|
+
"p4": -1,
|
152
|
+
"precision": 0,
|
153
|
+
"scale": 7,
|
154
|
+
"code_page": 0,
|
155
|
+
"p8": null,
|
156
|
+
"p9": null
|
157
|
+
},
|
158
|
+
{
|
159
|
+
"name": "NULL",
|
160
|
+
"data_type": 3,
|
161
|
+
"max_length": 4,
|
162
|
+
"p2": 104,
|
163
|
+
"p3": 48,
|
164
|
+
"p4": -1,
|
165
|
+
"precision": 0,
|
166
|
+
"scale": 0,
|
167
|
+
"code_page": 0,
|
168
|
+
"p8": null,
|
169
|
+
"p9": null
|
170
|
+
}
|
171
|
+
]
|
data/fixtures/sql11.raw
ADDED
Binary file
|
@@ -0,0 +1,3 @@
|
|
1
|
+
int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
|
2
|
+
1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 08:41:00.637 2016-01-12 08:41:01.0000000
|
3
|
+
2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 08:41:00.637 2016-01-12 08:41:00.6370000
|
@@ -0,0 +1,171 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"name": "int",
|
4
|
+
"data_type": 3,
|
5
|
+
"max_length": 4,
|
6
|
+
"p2": 52,
|
7
|
+
"p3": 0,
|
8
|
+
"p4": -1,
|
9
|
+
"precision": 0,
|
10
|
+
"scale": 0,
|
11
|
+
"code_page": 0,
|
12
|
+
"p8": 0,
|
13
|
+
"p9": 0
|
14
|
+
},
|
15
|
+
{
|
16
|
+
"name": "float",
|
17
|
+
"data_type": 5,
|
18
|
+
"max_length": 8,
|
19
|
+
"p2": 56,
|
20
|
+
"p3": 4,
|
21
|
+
"p4": -1,
|
22
|
+
"precision": 0,
|
23
|
+
"scale": 0,
|
24
|
+
"code_page": 0,
|
25
|
+
"p8": 0,
|
26
|
+
"p9": 0
|
27
|
+
},
|
28
|
+
{
|
29
|
+
"name": "bit",
|
30
|
+
"data_type": 11,
|
31
|
+
"max_length": 2,
|
32
|
+
"p2": 256,
|
33
|
+
"p3": 8,
|
34
|
+
"p4": -1,
|
35
|
+
"precision": 0,
|
36
|
+
"scale": 0,
|
37
|
+
"code_page": 0,
|
38
|
+
"p8": 0,
|
39
|
+
"p9": 0
|
40
|
+
},
|
41
|
+
{
|
42
|
+
"name": "bigint",
|
43
|
+
"data_type": 20,
|
44
|
+
"max_length": 8,
|
45
|
+
"p2": 96,
|
46
|
+
"p3": 12,
|
47
|
+
"p4": -1,
|
48
|
+
"precision": 0,
|
49
|
+
"scale": 0,
|
50
|
+
"code_page": 0,
|
51
|
+
"p8": 0,
|
52
|
+
"p9": 0
|
53
|
+
},
|
54
|
+
{
|
55
|
+
"name": "varchar",
|
56
|
+
"data_type": 129,
|
57
|
+
"max_length": 128,
|
58
|
+
"p2": 108,
|
59
|
+
"p3": 16,
|
60
|
+
"p4": -1,
|
61
|
+
"precision": 0,
|
62
|
+
"scale": 0,
|
63
|
+
"code_page": 1251,
|
64
|
+
"p8": 0,
|
65
|
+
"p9": 0
|
66
|
+
},
|
67
|
+
{
|
68
|
+
"name": "char",
|
69
|
+
"data_type": 129,
|
70
|
+
"max_length": 128,
|
71
|
+
"p2": 280,
|
72
|
+
"p3": 20,
|
73
|
+
"p4": -1,
|
74
|
+
"precision": 0,
|
75
|
+
"scale": 0,
|
76
|
+
"code_page": 1251,
|
77
|
+
"p8": 0,
|
78
|
+
"p9": 0
|
79
|
+
},
|
80
|
+
{
|
81
|
+
"name": "nvarchar",
|
82
|
+
"data_type": 130,
|
83
|
+
"max_length": 128,
|
84
|
+
"p2": 412,
|
85
|
+
"p3": 24,
|
86
|
+
"p4": -1,
|
87
|
+
"precision": 0,
|
88
|
+
"scale": 0,
|
89
|
+
"code_page": 0,
|
90
|
+
"p8": 0,
|
91
|
+
"p9": 0
|
92
|
+
},
|
93
|
+
{
|
94
|
+
"name": "nchar",
|
95
|
+
"data_type": 130,
|
96
|
+
"max_length": 128,
|
97
|
+
"p2": 672,
|
98
|
+
"p3": 28,
|
99
|
+
"p4": -1,
|
100
|
+
"precision": 0,
|
101
|
+
"scale": 0,
|
102
|
+
"code_page": 0,
|
103
|
+
"p8": 0,
|
104
|
+
"p9": 0
|
105
|
+
},
|
106
|
+
{
|
107
|
+
"name": "decimal",
|
108
|
+
"data_type": 131,
|
109
|
+
"max_length": 19,
|
110
|
+
"p2": 237,
|
111
|
+
"p3": 32,
|
112
|
+
"p4": -1,
|
113
|
+
"precision": 18,
|
114
|
+
"scale": 6,
|
115
|
+
"code_page": 0,
|
116
|
+
"p8": 0,
|
117
|
+
"p9": 0
|
118
|
+
},
|
119
|
+
{
|
120
|
+
"name": "numeric",
|
121
|
+
"data_type": 131,
|
122
|
+
"max_length": 19,
|
123
|
+
"p2": 258,
|
124
|
+
"p3": 36,
|
125
|
+
"p4": -1,
|
126
|
+
"precision": 18,
|
127
|
+
"scale": 6,
|
128
|
+
"code_page": 0,
|
129
|
+
"p8": 0,
|
130
|
+
"p9": 0
|
131
|
+
},
|
132
|
+
{
|
133
|
+
"name": "datetime",
|
134
|
+
"data_type": 135,
|
135
|
+
"max_length": 16,
|
136
|
+
"p2": 64,
|
137
|
+
"p3": 40,
|
138
|
+
"p4": -1,
|
139
|
+
"precision": 0,
|
140
|
+
"scale": 0,
|
141
|
+
"code_page": 0,
|
142
|
+
"p8": 0,
|
143
|
+
"p9": 0
|
144
|
+
},
|
145
|
+
{
|
146
|
+
"name": "datetime2",
|
147
|
+
"data_type": 304,
|
148
|
+
"max_length": 16,
|
149
|
+
"p2": 80,
|
150
|
+
"p3": 44,
|
151
|
+
"p4": -1,
|
152
|
+
"precision": 0,
|
153
|
+
"scale": 7,
|
154
|
+
"code_page": 0,
|
155
|
+
"p8": 0,
|
156
|
+
"p9": 0
|
157
|
+
},
|
158
|
+
{
|
159
|
+
"name": "NULL",
|
160
|
+
"data_type": 3,
|
161
|
+
"max_length": 4,
|
162
|
+
"p2": 104,
|
163
|
+
"p3": 48,
|
164
|
+
"p4": -1,
|
165
|
+
"precision": 0,
|
166
|
+
"scale": 0,
|
167
|
+
"code_page": 0,
|
168
|
+
"p8": 0,
|
169
|
+
"p9": 0
|
170
|
+
}
|
171
|
+
]
|
@@ -0,0 +1,162 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require "#{File.dirname(__FILE__)}/sql_server_dts_ns"
|
3
|
+
|
4
|
+
# SqlServerDts::RawFile class
|
5
|
+
class SqlServerDts::RawFile
|
6
|
+
# Error class
|
7
|
+
class Error < RuntimeError
|
8
|
+
end
|
9
|
+
|
10
|
+
# UnsupportedRawVersionError class
|
11
|
+
class UnsupportedRawVersionError < Error
|
12
|
+
end
|
13
|
+
|
14
|
+
# UnknownDataTypeError class
|
15
|
+
class UnknownDataTypeError < Error
|
16
|
+
end
|
17
|
+
|
18
|
+
# FieldsNotExistsError class
|
19
|
+
class FieldsNotExistsError < Error
|
20
|
+
end
|
21
|
+
|
22
|
+
# default encoding of raw file: 'utf-16le'
|
23
|
+
DEFAULT_RAW_FILE_ENCODING = 'utf-16le'
|
24
|
+
|
25
|
+
# default encoding of module environment: 'utf-8'
|
26
|
+
DEFAULT_ENCODING = 'utf-8'
|
27
|
+
|
28
|
+
# supported versions of Raw File:
|
29
|
+
# 00.90.00.00 => 9 (SQL Server: 9/10)
|
30
|
+
# 00.10.01.00 => 10 (SQL Server: 11)
|
31
|
+
SUPPORTED_RAW_VERSIONS = %w(00.10.01.00 00.90.00.00)
|
32
|
+
|
33
|
+
# initialization
|
34
|
+
#
|
35
|
+
# @param io [String, IO] Path to raw file or io-stream
|
36
|
+
# @param init_pos [Integer, nil] Initial position in raw file or nil
|
37
|
+
# @return [SqlServerDts::RawFile, [SqlServerDts::RawFile,IO]] Within block it returns self and io-stream
|
38
|
+
# in other side returns self
|
39
|
+
def initialize(io,init_pos=nil)
|
40
|
+
if io.is_a?(IO)
|
41
|
+
@raw_file = io
|
42
|
+
@raw_file.binmode unless @raw_file.binmode?
|
43
|
+
read_header
|
44
|
+
@raw_file.seek(init_pos || @data_pos)
|
45
|
+
else
|
46
|
+
@raw_file = File.new(io,'rb')
|
47
|
+
read_header
|
48
|
+
@raw_file.seek(init_pos || @data_pos)
|
49
|
+
if block_given?
|
50
|
+
begin
|
51
|
+
yield [self,@raw_file]
|
52
|
+
ensure
|
53
|
+
@raw_file.close
|
54
|
+
end
|
55
|
+
else
|
56
|
+
self
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# read metadata (header) raw file and fill inner array of metadata
|
62
|
+
def read_header
|
63
|
+
raise IOError, 'Stream is closed' if @raw_file.closed?
|
64
|
+
begin
|
65
|
+
@raw_file.rewind
|
66
|
+
@version = @raw_file.read(4).unpack('H2H2H2H2').join('.')
|
67
|
+
raise UnsupportedRawVersionError, "Version #{@version} unsupported " unless SUPPORTED_RAW_VERSIONS.include?(@version)
|
68
|
+
@fields_info = []
|
69
|
+
fields_count = @raw_file.read(4).unpack('I')[0]
|
70
|
+
raise FieldsNotExistsError, 'Fields not exists (fields_count=0)' if fields_count==0
|
71
|
+
@nil_mask_len = fields_count/8+(fields_count-fields_count/8*8>0?1:0)
|
72
|
+
fields_count.times {
|
73
|
+
name_len = @raw_file.read(4).unpack('I')[0]
|
74
|
+
field_name = @raw_file.read(name_len*2).encode(DEFAULT_ENCODING,DEFAULT_RAW_FILE_ENCODING,{ invalid: :replace, undef: :replace })
|
75
|
+
(@version == '00.10.01.00') ?
|
76
|
+
values_arr = @raw_file.read(40).unpack('I4iI5') :
|
77
|
+
values_arr = @raw_file.read(32).unpack('I4iI3')
|
78
|
+
field_info = {
|
79
|
+
name: field_name,
|
80
|
+
data_type: values_arr[0],
|
81
|
+
max_length: values_arr[1],
|
82
|
+
p2: values_arr[2],
|
83
|
+
p3: values_arr[3],
|
84
|
+
p4: values_arr[4],
|
85
|
+
precision: values_arr[5],
|
86
|
+
scale: values_arr[6],
|
87
|
+
code_page: values_arr[7],
|
88
|
+
p8: values_arr[8],
|
89
|
+
p9: values_arr[9]
|
90
|
+
}
|
91
|
+
@fields_info << field_info
|
92
|
+
}
|
93
|
+
rescue
|
94
|
+
raise
|
95
|
+
else
|
96
|
+
@data_pos = @raw_file.pos
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# read line of raw data
|
101
|
+
#
|
102
|
+
# @return [Hash] Hash of converted raw data (one line)
|
103
|
+
def read_line
|
104
|
+
raise IOError, 'Stream is closed' if @raw_file.closed?
|
105
|
+
begin
|
106
|
+
nil_mask = @raw_file.read(@nil_mask_len).unpack('b*')[0].each_char.to_a
|
107
|
+
data_row = []
|
108
|
+
@fields_info.each_with_index { |field_info, index|
|
109
|
+
field_value =
|
110
|
+
if nil_mask[index] == '1'
|
111
|
+
nil
|
112
|
+
else
|
113
|
+
case field_info[:data_type]
|
114
|
+
when 3 # int, NULL w/o cast
|
115
|
+
@raw_file.read(field_info[:max_length]).unpack('I')[0]
|
116
|
+
when 5 # float
|
117
|
+
@raw_file.read(field_info[:max_length]).unpack('D')[0]
|
118
|
+
when 11 # bit
|
119
|
+
(@raw_file.read(field_info[:max_length]).unpack('B')[0] == '1') ? true : false
|
120
|
+
when 20 # bigint
|
121
|
+
@raw_file.read(field_info[:max_length]).unpack('L')[0]
|
122
|
+
when 129 # varchar, char
|
123
|
+
data_len = @raw_file.read(4).unpack('I')[0]
|
124
|
+
@raw_file.read(data_len).encode({invalid: :replace, undef: :replace})
|
125
|
+
when 130 # nvarchar, nchar
|
126
|
+
data_len = @raw_file.read(4).unpack('I')[0]
|
127
|
+
@raw_file.read(data_len*2).encode(DEFAULT_ENCODING, DEFAULT_RAW_FILE_ENCODING, {invalid: :replace, undef: :replace})
|
128
|
+
when 131 # numeric, decimal
|
129
|
+
# precision, scale, sign, value
|
130
|
+
values_arr = @raw_file.read(field_info[:max_length]).unpack('C3Q')
|
131
|
+
"#{('-' if values_arr[2] == 0)}#{values_arr[3].to_s.insert((values_arr[3].to_s.length-values_arr[1]), '.')}"
|
132
|
+
when 135 # datetime
|
133
|
+
values_arr = @raw_file.read(field_info[:max_length]).unpack('S6I')
|
134
|
+
Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime('%F %T.%L')
|
135
|
+
when 304 # datetime2
|
136
|
+
values_arr = @raw_file.read(field_info[:max_length]).unpack('S6I')
|
137
|
+
Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime("%F %T.%#{field_info[:scale]}N")
|
138
|
+
else
|
139
|
+
raise UnknownDataTypeError, "Unknown data type #{field_info[:data_type]}"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
data_row << [ field_info[:name], field_value ]
|
143
|
+
}
|
144
|
+
rescue
|
145
|
+
raise
|
146
|
+
else
|
147
|
+
data_row.to_h
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
# get metadata (header) of raw file
|
152
|
+
#
|
153
|
+
# @return [Array] Array of hash of fields metadata
|
154
|
+
def header
|
155
|
+
@fields_info
|
156
|
+
end
|
157
|
+
|
158
|
+
# get version of raw file
|
159
|
+
def version
|
160
|
+
@version
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
require "#{File.dirname(__FILE__)}/lib/sql_server_dts_ns"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'sql-server-raw-file'
|
7
|
+
s.version = SqlServerDts::VERSION
|
8
|
+
s.licenses = ['MIT']
|
9
|
+
s.summary = 'sql-server-raw-file lib'
|
10
|
+
s.description = 'Converts SQL Server Integration Services Raw File source to hash.'
|
11
|
+
s.authors = ['Dmitriy Mullo']
|
12
|
+
s.email = ['d.a.mullo1981@gmail.com']
|
13
|
+
s.homepage = 'https://github.com/dim11981/sql-server-raw-file'
|
14
|
+
s.platform = Gem::Platform::RUBY
|
15
|
+
s.files = Dir['*.md']+Dir['sql-server-raw-file.*']+Dir['lib/*.rb']+Dir['test/*.rb']+Dir['fixtures/*']
|
16
|
+
s.require_path = 'lib'
|
17
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require 'csv'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
require "#{File.dirname(__FILE__)}/../lib/sql_server_raw_file"
|
6
|
+
|
7
|
+
fixtures_path = "#{File.dirname(__FILE__)}/../fixtures/"
|
8
|
+
raw_path = %w(sql09.raw sql10.raw sql11.raw)
|
9
|
+
|
10
|
+
# current_fixture_index = 0
|
11
|
+
current_fixture_index = File.read(fixtures_path+'/current_sql_index').to_i
|
12
|
+
|
13
|
+
# show header of raw file and convert it hash and put it json-file
|
14
|
+
File.open(fixtures_path+raw_path[current_fixture_index]) { |raw_io|
|
15
|
+
raw_obj = SqlServerDts::RawFile.new(raw_io)
|
16
|
+
puts "RAW VERSION: #{raw_obj.version}"
|
17
|
+
puts '=== BEGIN HEADER ==='
|
18
|
+
puts raw_obj.header
|
19
|
+
File.open("#{fixtures_path+File.basename(raw_io.path, '.*')}_header.json",'wt') { |json_io|
|
20
|
+
header_json = JSON.pretty_generate(raw_obj.header)
|
21
|
+
json_io.write(header_json)
|
22
|
+
}
|
23
|
+
puts '=== END HEADER ==='
|
24
|
+
}
|
25
|
+
|
26
|
+
# show data of raw file in rows table and convert it hash and put it csv-file
|
27
|
+
SqlServerDts::RawFile.new(fixtures_path+raw_path[current_fixture_index]) { |raw_obj, raw_io|
|
28
|
+
puts '=== BEGIN DATA ==='
|
29
|
+
i=0
|
30
|
+
fields_arr = []
|
31
|
+
raw_obj.header.each { |field_info| fields_arr << field_info[:name] }
|
32
|
+
puts fields_arr.join("\t")
|
33
|
+
CSV.open("#{fixtures_path+File.basename(raw_io.path, '.*')}_data.csv", 'wb', {headers: fields_arr, write_headers: true, col_sep: "\t"}) { |csv|
|
34
|
+
until raw_io.eof?
|
35
|
+
values_arr = []
|
36
|
+
raw_obj.read_line.each_value { |v| values_arr << v }
|
37
|
+
csv << values_arr
|
38
|
+
puts values_arr.join("\t")
|
39
|
+
i+=1
|
40
|
+
end
|
41
|
+
}
|
42
|
+
puts '=== END DATA ==='
|
43
|
+
puts "total: #{i}"
|
44
|
+
}
|
metadata
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sql-server-raw-file
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dmitriy Mullo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-01-14 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Converts SQL Server Integration Services Raw File source to hash.
|
14
|
+
email:
|
15
|
+
- d.a.mullo1981@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- CHANGELOG.md
|
21
|
+
- LICENSE.md
|
22
|
+
- README.md
|
23
|
+
- fixtures/current_sql_index
|
24
|
+
- fixtures/select-fixture-data-SQL_Server_90.sql
|
25
|
+
- fixtures/select-fixture-data.sql
|
26
|
+
- fixtures/sql09.raw
|
27
|
+
- fixtures/sql09_data.csv
|
28
|
+
- fixtures/sql09_header.json
|
29
|
+
- fixtures/sql10.raw
|
30
|
+
- fixtures/sql10_data.csv
|
31
|
+
- fixtures/sql10_header.json
|
32
|
+
- fixtures/sql11.raw
|
33
|
+
- fixtures/sql11_data.csv
|
34
|
+
- fixtures/sql11_header.json
|
35
|
+
- lib/sql_server_dts_ns.rb
|
36
|
+
- lib/sql_server_raw_file.rb
|
37
|
+
- sql-server-raw-file.gemspec
|
38
|
+
- test/test_sql_data_and_header.rb
|
39
|
+
homepage: https://github.com/dim11981/sql-server-raw-file
|
40
|
+
licenses:
|
41
|
+
- MIT
|
42
|
+
metadata: {}
|
43
|
+
post_install_message:
|
44
|
+
rdoc_options: []
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
48
|
+
requirements:
|
49
|
+
- - ">="
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: '0'
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '0'
|
57
|
+
requirements: []
|
58
|
+
rubyforge_project:
|
59
|
+
rubygems_version: 2.4.5.1
|
60
|
+
signing_key:
|
61
|
+
specification_version: 4
|
62
|
+
summary: sql-server-raw-file lib
|
63
|
+
test_files: []
|