sql-server-raw-file 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: ea75b306d331a3e23df7a321c868778e60266a13
4
+ data.tar.gz: b79c539f190cc42633fbca1dd980a7a878e60b88
5
+ SHA512:
6
+ metadata.gz: ad451f419c448ca2bc864875c4141c6931457e0bdf73113f1e5fb9ec170e2f30a882c9358dc9e0868d19bca2df7e627ae3bcbb36838eb746b324aef7ef36fe39
7
+ data.tar.gz: f415ea6b1439db87d8a930dec480d5eb33bb2fe74d1f982e79944af80dd1abad365edc34025b2bad2cef3fd6651096a119ee3bd027b4f290334a5b7fc1a67cb7
data/CHANGELOG.md ADDED
@@ -0,0 +1,20 @@
1
+ ## 0.0.1 (Jan 5, 2016)
2
+
3
+ Features:
4
+
5
+ - added support of SQL Server Dts Raw File versions:
6
+ * 00.90.00.00 => 9 (SQL Server: 9/10)
7
+ * 00.10.01.00 => 10 (SQL Server: 11)
8
+ - added support of basic data types:
9
+ * int (i.e. NULL w/o cast)
10
+ * float
11
+ * bit
12
+ * bigint
13
+ * (n)(var)char
14
+ * decimal, numeric
15
+ * datetime
16
+ * datetime2 (only SQL Server 10/11)
17
+
18
+ Bugfixes:
19
+
20
+ - initial release
data/LICENSE.md ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2016 Dmitriy Mullo
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,50 @@
1
+ # sql-server-raw-file: sql-server-raw-file lib
2
+
3
+ Converts SQL Server Dts Raw File source to hash.
4
+
5
+ ### Installation and usage
6
+
7
+ ```
8
+ 1) gem install sql-server-raw-file
9
+ 2) usage (examples):
10
+
11
+ # begin example 1
12
+ require 'sql_server_raw_file'
13
+
14
+ File.open('/path/to/sql/raw/file') { |raw_io|
15
+ raw_obj = SqlServerDts::RawFile.new(raw_io)
16
+ # ...
17
+ puts raw_obj.version
18
+ puts raw_obj.header
19
+ # ...
20
+ until raw_io.eof?
21
+ # ...
22
+ puts raw_obj.read_line
23
+ # ...
24
+ end
25
+ # ...
26
+ }
27
+ # end example 1
28
+
29
+ # begin example 2
30
+ require 'sql_server_raw_file'
31
+
32
+ SqlServerDts::RawFile.new('/path/to/sql/raw/file') { |raw_obj, raw_io|
33
+ # ...
34
+ puts raw_obj.version
35
+ puts raw_obj.header
36
+ # ...
37
+ until raw_io.eof?
38
+ # ...
39
+ puts raw_obj.read_line
40
+ # ...
41
+ end
42
+ # ...
43
+ }
44
+ # end example 2
45
+ ```
46
+
47
+ ### Troubleshooting
48
+
49
+ mailto:d.a.mullo1981@gmail.com
50
+ subject: sql-server-raw-file issue
@@ -0,0 +1 @@
1
+ 2
@@ -0,0 +1,3 @@
1
+ select cast(1 as int) [int], cast(3.14159265 as float) [float], cast(1 as bit) [bit], cast(POWER(2,30) as bigint) [bigint], cast('This is a variable length char string' as varchar(128)) [varchar], cast('This is a char string' as char(128)) [char], cast('��� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ������ � �������' as nchar(128)) [nchar], cast(9.11 as decimal(18,6)) [decimal], cast(7.62 as numeric(18,6)) [numeric], getdate() [datetime], NULL [datetime2], NULL AS [NULL]
2
+ UNION ALL
3
+ select cast(2 as int) [int], cast(-37.0/3.0 as float) [float], cast(0 as bit) [bit], cast(-POWER(3,19) as bigint) [bigint], cast('This is another variable length char string' as varchar(128)) [varchar], cast('This is another char string' as char(128)) [char], cast('��� ��� ���� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ��� ���� ������ � �������' as nchar(128)) [nchar], cast(-99.999 as decimal(18,6)) [decimal], cast(-5.56 as numeric(18,6)) [numeric], getdate() [datetime], NULL [datetime2], NULL AS [NULL]
@@ -0,0 +1,3 @@
1
+ select cast(1 as int) [int], cast(3.14159265 as float) [float], cast(1 as bit) [bit], cast(POWER(2,30) as bigint) [bigint], cast('This is a variable length char string' as varchar(128)) [varchar], cast('This is a char string' as char(128)) [char], cast('��� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ������ � �������' as nchar(128)) [nchar], cast(9.11 as decimal(18,6)) [decimal], cast(7.62 as numeric(18,6)) [numeric], getdate() [datetime], CAST(GETDATE() AS DATETIME2(0)) [datetime2], NULL AS [NULL]
2
+ UNION ALL
3
+ select cast(2 as int) [int], cast(-37.0/3.0 as float) [float], cast(0 as bit) [bit], cast(-POWER(3,19) as bigint) [bigint], cast('This is another variable length char string' as varchar(128)) [varchar], cast('This is another char string' as char(128)) [char], cast('��� ��� ���� ������ ���������� ����� � �������' as nvarchar(128)) [nvarchar], cast('��� ��� ���� ������ � �������' as nchar(128)) [nchar], cast(-99.999 as decimal(18,6)) [decimal], cast(-5.56 as numeric(18,6)) [numeric], getdate() [datetime], CAST(GETDATE() AS DATETIME2(7)) [datetime2], NULL AS [NULL]
Binary file
@@ -0,0 +1,3 @@
1
+ int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
2
+ 1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 09:26:52.747
3
+ 2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 09:26:52.747
@@ -0,0 +1,171 @@
1
+ [
2
+ {
3
+ "name": "int",
4
+ "data_type": 3,
5
+ "max_length": 4,
6
+ "p2": 52,
7
+ "p3": 0,
8
+ "p4": -1,
9
+ "precision": 0,
10
+ "scale": 0,
11
+ "code_page": 0,
12
+ "p8": null,
13
+ "p9": null
14
+ },
15
+ {
16
+ "name": "float",
17
+ "data_type": 5,
18
+ "max_length": 8,
19
+ "p2": 56,
20
+ "p3": 4,
21
+ "p4": -1,
22
+ "precision": 0,
23
+ "scale": 0,
24
+ "code_page": 0,
25
+ "p8": null,
26
+ "p9": null
27
+ },
28
+ {
29
+ "name": "bit",
30
+ "data_type": 11,
31
+ "max_length": 2,
32
+ "p2": 392,
33
+ "p3": 8,
34
+ "p4": -1,
35
+ "precision": 0,
36
+ "scale": 0,
37
+ "code_page": 0,
38
+ "p8": null,
39
+ "p9": null
40
+ },
41
+ {
42
+ "name": "bigint",
43
+ "data_type": 20,
44
+ "max_length": 8,
45
+ "p2": 80,
46
+ "p3": 12,
47
+ "p4": -1,
48
+ "precision": 0,
49
+ "scale": 0,
50
+ "code_page": 0,
51
+ "p8": null,
52
+ "p9": null
53
+ },
54
+ {
55
+ "name": "varchar",
56
+ "data_type": 129,
57
+ "max_length": 128,
58
+ "p2": 92,
59
+ "p3": 16,
60
+ "p4": -1,
61
+ "precision": 0,
62
+ "scale": 0,
63
+ "code_page": 1251,
64
+ "p8": null,
65
+ "p9": null
66
+ },
67
+ {
68
+ "name": "char",
69
+ "data_type": 129,
70
+ "max_length": 128,
71
+ "p2": 244,
72
+ "p3": 20,
73
+ "p4": -1,
74
+ "precision": 0,
75
+ "scale": 0,
76
+ "code_page": 1251,
77
+ "p8": null,
78
+ "p9": null
79
+ },
80
+ {
81
+ "name": "nvarchar",
82
+ "data_type": 130,
83
+ "max_length": 128,
84
+ "p2": 396,
85
+ "p3": 24,
86
+ "p4": -1,
87
+ "precision": 0,
88
+ "scale": 0,
89
+ "code_page": 0,
90
+ "p8": null,
91
+ "p9": null
92
+ },
93
+ {
94
+ "name": "nchar",
95
+ "data_type": 130,
96
+ "max_length": 128,
97
+ "p2": 656,
98
+ "p3": 28,
99
+ "p4": -1,
100
+ "precision": 0,
101
+ "scale": 0,
102
+ "code_page": 0,
103
+ "p8": null,
104
+ "p9": null
105
+ },
106
+ {
107
+ "name": "decimal",
108
+ "data_type": 131,
109
+ "max_length": 19,
110
+ "p2": 221,
111
+ "p3": 32,
112
+ "p4": -1,
113
+ "precision": 18,
114
+ "scale": 6,
115
+ "code_page": 0,
116
+ "p8": null,
117
+ "p9": null
118
+ },
119
+ {
120
+ "name": "numeric",
121
+ "data_type": 131,
122
+ "max_length": 19,
123
+ "p2": 373,
124
+ "p3": 36,
125
+ "p4": -1,
126
+ "precision": 18,
127
+ "scale": 6,
128
+ "code_page": 0,
129
+ "p8": null,
130
+ "p9": null
131
+ },
132
+ {
133
+ "name": "datetime",
134
+ "data_type": 135,
135
+ "max_length": 16,
136
+ "p2": 64,
137
+ "p3": 40,
138
+ "p4": -1,
139
+ "precision": 0,
140
+ "scale": 0,
141
+ "code_page": 0,
142
+ "p8": null,
143
+ "p9": null
144
+ },
145
+ {
146
+ "name": "datetime2",
147
+ "data_type": 3,
148
+ "max_length": 4,
149
+ "p2": 88,
150
+ "p3": 44,
151
+ "p4": -1,
152
+ "precision": 0,
153
+ "scale": 0,
154
+ "code_page": 0,
155
+ "p8": null,
156
+ "p9": null
157
+ },
158
+ {
159
+ "name": "NULL",
160
+ "data_type": 3,
161
+ "max_length": 4,
162
+ "p2": 240,
163
+ "p3": 48,
164
+ "p4": -1,
165
+ "precision": 0,
166
+ "scale": 0,
167
+ "code_page": 0,
168
+ "p8": null,
169
+ "p9": null
170
+ }
171
+ ]
Binary file
@@ -0,0 +1,3 @@
1
+ int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
2
+ 1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 08:20:06.823 2016-01-12 08:20:07.0000000
3
+ 2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 08:20:06.823 2016-01-12 08:20:06.8230000
@@ -0,0 +1,171 @@
1
+ [
2
+ {
3
+ "name": "int",
4
+ "data_type": 3,
5
+ "max_length": 4,
6
+ "p2": 52,
7
+ "p3": 0,
8
+ "p4": -1,
9
+ "precision": 0,
10
+ "scale": 0,
11
+ "code_page": 0,
12
+ "p8": null,
13
+ "p9": null
14
+ },
15
+ {
16
+ "name": "float",
17
+ "data_type": 5,
18
+ "max_length": 8,
19
+ "p2": 56,
20
+ "p3": 4,
21
+ "p4": -1,
22
+ "precision": 0,
23
+ "scale": 0,
24
+ "code_page": 0,
25
+ "p8": null,
26
+ "p9": null
27
+ },
28
+ {
29
+ "name": "bit",
30
+ "data_type": 11,
31
+ "max_length": 2,
32
+ "p2": 256,
33
+ "p3": 8,
34
+ "p4": -1,
35
+ "precision": 0,
36
+ "scale": 0,
37
+ "code_page": 0,
38
+ "p8": null,
39
+ "p9": null
40
+ },
41
+ {
42
+ "name": "bigint",
43
+ "data_type": 20,
44
+ "max_length": 8,
45
+ "p2": 96,
46
+ "p3": 12,
47
+ "p4": -1,
48
+ "precision": 0,
49
+ "scale": 0,
50
+ "code_page": 0,
51
+ "p8": null,
52
+ "p9": null
53
+ },
54
+ {
55
+ "name": "varchar",
56
+ "data_type": 129,
57
+ "max_length": 128,
58
+ "p2": 108,
59
+ "p3": 16,
60
+ "p4": -1,
61
+ "precision": 0,
62
+ "scale": 0,
63
+ "code_page": 1251,
64
+ "p8": null,
65
+ "p9": null
66
+ },
67
+ {
68
+ "name": "char",
69
+ "data_type": 129,
70
+ "max_length": 128,
71
+ "p2": 280,
72
+ "p3": 20,
73
+ "p4": -1,
74
+ "precision": 0,
75
+ "scale": 0,
76
+ "code_page": 1251,
77
+ "p8": null,
78
+ "p9": null
79
+ },
80
+ {
81
+ "name": "nvarchar",
82
+ "data_type": 130,
83
+ "max_length": 128,
84
+ "p2": 412,
85
+ "p3": 24,
86
+ "p4": -1,
87
+ "precision": 0,
88
+ "scale": 0,
89
+ "code_page": 0,
90
+ "p8": null,
91
+ "p9": null
92
+ },
93
+ {
94
+ "name": "nchar",
95
+ "data_type": 130,
96
+ "max_length": 128,
97
+ "p2": 672,
98
+ "p3": 28,
99
+ "p4": -1,
100
+ "precision": 0,
101
+ "scale": 0,
102
+ "code_page": 0,
103
+ "p8": null,
104
+ "p9": null
105
+ },
106
+ {
107
+ "name": "decimal",
108
+ "data_type": 131,
109
+ "max_length": 19,
110
+ "p2": 237,
111
+ "p3": 32,
112
+ "p4": -1,
113
+ "precision": 18,
114
+ "scale": 6,
115
+ "code_page": 0,
116
+ "p8": null,
117
+ "p9": null
118
+ },
119
+ {
120
+ "name": "numeric",
121
+ "data_type": 131,
122
+ "max_length": 19,
123
+ "p2": 258,
124
+ "p3": 36,
125
+ "p4": -1,
126
+ "precision": 18,
127
+ "scale": 6,
128
+ "code_page": 0,
129
+ "p8": null,
130
+ "p9": null
131
+ },
132
+ {
133
+ "name": "datetime",
134
+ "data_type": 135,
135
+ "max_length": 16,
136
+ "p2": 64,
137
+ "p3": 40,
138
+ "p4": -1,
139
+ "precision": 0,
140
+ "scale": 0,
141
+ "code_page": 0,
142
+ "p8": null,
143
+ "p9": null
144
+ },
145
+ {
146
+ "name": "datetime2",
147
+ "data_type": 304,
148
+ "max_length": 16,
149
+ "p2": 80,
150
+ "p3": 44,
151
+ "p4": -1,
152
+ "precision": 0,
153
+ "scale": 7,
154
+ "code_page": 0,
155
+ "p8": null,
156
+ "p9": null
157
+ },
158
+ {
159
+ "name": "NULL",
160
+ "data_type": 3,
161
+ "max_length": 4,
162
+ "p2": 104,
163
+ "p3": 48,
164
+ "p4": -1,
165
+ "precision": 0,
166
+ "scale": 0,
167
+ "code_page": 0,
168
+ "p8": null,
169
+ "p9": null
170
+ }
171
+ ]
Binary file
@@ -0,0 +1,3 @@
1
+ int float bit bigint varchar char nvarchar nchar decimal numeric datetime datetime2 NULL
2
+ 1 3.14159265 true 1073741824 This is a variable length char string This is a char string Это строка переменной длины в Юникоде Это строка в Юникоде 9.110000 7.620000 2016-01-12 08:41:00.637 2016-01-12 08:41:01.0000000
3
+ 2 -12.333333 false 3132705829 This is another variable length char string This is another char string Это еще одна строка переменной длины в Юникоде Это еще одна строка в Юникоде -99.999000 -5.560000 2016-01-12 08:41:00.637 2016-01-12 08:41:00.6370000
@@ -0,0 +1,171 @@
1
+ [
2
+ {
3
+ "name": "int",
4
+ "data_type": 3,
5
+ "max_length": 4,
6
+ "p2": 52,
7
+ "p3": 0,
8
+ "p4": -1,
9
+ "precision": 0,
10
+ "scale": 0,
11
+ "code_page": 0,
12
+ "p8": 0,
13
+ "p9": 0
14
+ },
15
+ {
16
+ "name": "float",
17
+ "data_type": 5,
18
+ "max_length": 8,
19
+ "p2": 56,
20
+ "p3": 4,
21
+ "p4": -1,
22
+ "precision": 0,
23
+ "scale": 0,
24
+ "code_page": 0,
25
+ "p8": 0,
26
+ "p9": 0
27
+ },
28
+ {
29
+ "name": "bit",
30
+ "data_type": 11,
31
+ "max_length": 2,
32
+ "p2": 256,
33
+ "p3": 8,
34
+ "p4": -1,
35
+ "precision": 0,
36
+ "scale": 0,
37
+ "code_page": 0,
38
+ "p8": 0,
39
+ "p9": 0
40
+ },
41
+ {
42
+ "name": "bigint",
43
+ "data_type": 20,
44
+ "max_length": 8,
45
+ "p2": 96,
46
+ "p3": 12,
47
+ "p4": -1,
48
+ "precision": 0,
49
+ "scale": 0,
50
+ "code_page": 0,
51
+ "p8": 0,
52
+ "p9": 0
53
+ },
54
+ {
55
+ "name": "varchar",
56
+ "data_type": 129,
57
+ "max_length": 128,
58
+ "p2": 108,
59
+ "p3": 16,
60
+ "p4": -1,
61
+ "precision": 0,
62
+ "scale": 0,
63
+ "code_page": 1251,
64
+ "p8": 0,
65
+ "p9": 0
66
+ },
67
+ {
68
+ "name": "char",
69
+ "data_type": 129,
70
+ "max_length": 128,
71
+ "p2": 280,
72
+ "p3": 20,
73
+ "p4": -1,
74
+ "precision": 0,
75
+ "scale": 0,
76
+ "code_page": 1251,
77
+ "p8": 0,
78
+ "p9": 0
79
+ },
80
+ {
81
+ "name": "nvarchar",
82
+ "data_type": 130,
83
+ "max_length": 128,
84
+ "p2": 412,
85
+ "p3": 24,
86
+ "p4": -1,
87
+ "precision": 0,
88
+ "scale": 0,
89
+ "code_page": 0,
90
+ "p8": 0,
91
+ "p9": 0
92
+ },
93
+ {
94
+ "name": "nchar",
95
+ "data_type": 130,
96
+ "max_length": 128,
97
+ "p2": 672,
98
+ "p3": 28,
99
+ "p4": -1,
100
+ "precision": 0,
101
+ "scale": 0,
102
+ "code_page": 0,
103
+ "p8": 0,
104
+ "p9": 0
105
+ },
106
+ {
107
+ "name": "decimal",
108
+ "data_type": 131,
109
+ "max_length": 19,
110
+ "p2": 237,
111
+ "p3": 32,
112
+ "p4": -1,
113
+ "precision": 18,
114
+ "scale": 6,
115
+ "code_page": 0,
116
+ "p8": 0,
117
+ "p9": 0
118
+ },
119
+ {
120
+ "name": "numeric",
121
+ "data_type": 131,
122
+ "max_length": 19,
123
+ "p2": 258,
124
+ "p3": 36,
125
+ "p4": -1,
126
+ "precision": 18,
127
+ "scale": 6,
128
+ "code_page": 0,
129
+ "p8": 0,
130
+ "p9": 0
131
+ },
132
+ {
133
+ "name": "datetime",
134
+ "data_type": 135,
135
+ "max_length": 16,
136
+ "p2": 64,
137
+ "p3": 40,
138
+ "p4": -1,
139
+ "precision": 0,
140
+ "scale": 0,
141
+ "code_page": 0,
142
+ "p8": 0,
143
+ "p9": 0
144
+ },
145
+ {
146
+ "name": "datetime2",
147
+ "data_type": 304,
148
+ "max_length": 16,
149
+ "p2": 80,
150
+ "p3": 44,
151
+ "p4": -1,
152
+ "precision": 0,
153
+ "scale": 7,
154
+ "code_page": 0,
155
+ "p8": 0,
156
+ "p9": 0
157
+ },
158
+ {
159
+ "name": "NULL",
160
+ "data_type": 3,
161
+ "max_length": 4,
162
+ "p2": 104,
163
+ "p3": 48,
164
+ "p4": -1,
165
+ "precision": 0,
166
+ "scale": 0,
167
+ "code_page": 0,
168
+ "p8": 0,
169
+ "p9": 0
170
+ }
171
+ ]
@@ -0,0 +1,5 @@
1
+ # SqlServerDts module
2
+ module SqlServerDts
3
+ # current version
4
+ VERSION = '0.0.1'
5
+ end
@@ -0,0 +1,162 @@
1
+ # coding: utf-8
2
+ require "#{File.dirname(__FILE__)}/sql_server_dts_ns"
3
+
4
+ # SqlServerDts::RawFile class
5
+ class SqlServerDts::RawFile
6
+ # Error class
7
+ class Error < RuntimeError
8
+ end
9
+
10
+ # UnsupportedRawVersionError class
11
+ class UnsupportedRawVersionError < Error
12
+ end
13
+
14
+ # UnknownDataTypeError class
15
+ class UnknownDataTypeError < Error
16
+ end
17
+
18
+ # FieldsNotExistsError class
19
+ class FieldsNotExistsError < Error
20
+ end
21
+
22
+ # default encoding of raw file: 'utf-16le'
23
+ DEFAULT_RAW_FILE_ENCODING = 'utf-16le'
24
+
25
+ # default encoding of module environment: 'utf-8'
26
+ DEFAULT_ENCODING = 'utf-8'
27
+
28
+ # supported versions of Raw File:
29
+ # 00.90.00.00 => 9 (SQL Server: 9/10)
30
+ # 00.10.01.00 => 10 (SQL Server: 11)
31
+ SUPPORTED_RAW_VERSIONS = %w(00.10.01.00 00.90.00.00)
32
+
33
+ # initialization
34
+ #
35
+ # @param io [String, IO] Path to raw file or io-stream
36
+ # @param init_pos [Integer, nil] Initial position in raw file or nil
37
+ # @return [SqlServerDts::RawFile, [SqlServerDts::RawFile,IO]] Within block it returns self and io-stream
38
+ # in other side returns self
39
+ def initialize(io,init_pos=nil)
40
+ if io.is_a?(IO)
41
+ @raw_file = io
42
+ @raw_file.binmode unless @raw_file.binmode?
43
+ read_header
44
+ @raw_file.seek(init_pos || @data_pos)
45
+ else
46
+ @raw_file = File.new(io,'rb')
47
+ read_header
48
+ @raw_file.seek(init_pos || @data_pos)
49
+ if block_given?
50
+ begin
51
+ yield [self,@raw_file]
52
+ ensure
53
+ @raw_file.close
54
+ end
55
+ else
56
+ self
57
+ end
58
+ end
59
+ end
60
+
61
+ # read metadata (header) raw file and fill inner array of metadata
62
+ def read_header
63
+ raise IOError, 'Stream is closed' if @raw_file.closed?
64
+ begin
65
+ @raw_file.rewind
66
+ @version = @raw_file.read(4).unpack('H2H2H2H2').join('.')
67
+ raise UnsupportedRawVersionError, "Version #{@version} unsupported " unless SUPPORTED_RAW_VERSIONS.include?(@version)
68
+ @fields_info = []
69
+ fields_count = @raw_file.read(4).unpack('I')[0]
70
+ raise FieldsNotExistsError, 'Fields not exists (fields_count=0)' if fields_count==0
71
+ @nil_mask_len = fields_count/8+(fields_count-fields_count/8*8>0?1:0)
72
+ fields_count.times {
73
+ name_len = @raw_file.read(4).unpack('I')[0]
74
+ field_name = @raw_file.read(name_len*2).encode(DEFAULT_ENCODING,DEFAULT_RAW_FILE_ENCODING,{ invalid: :replace, undef: :replace })
75
+ (@version == '00.10.01.00') ?
76
+ values_arr = @raw_file.read(40).unpack('I4iI5') :
77
+ values_arr = @raw_file.read(32).unpack('I4iI3')
78
+ field_info = {
79
+ name: field_name,
80
+ data_type: values_arr[0],
81
+ max_length: values_arr[1],
82
+ p2: values_arr[2],
83
+ p3: values_arr[3],
84
+ p4: values_arr[4],
85
+ precision: values_arr[5],
86
+ scale: values_arr[6],
87
+ code_page: values_arr[7],
88
+ p8: values_arr[8],
89
+ p9: values_arr[9]
90
+ }
91
+ @fields_info << field_info
92
+ }
93
+ rescue
94
+ raise
95
+ else
96
+ @data_pos = @raw_file.pos
97
+ end
98
+ end
99
+
100
+ # read line of raw data
101
+ #
102
+ # @return [Hash] Hash of converted raw data (one line)
103
+ def read_line
104
+ raise IOError, 'Stream is closed' if @raw_file.closed?
105
+ begin
106
+ nil_mask = @raw_file.read(@nil_mask_len).unpack('b*')[0].each_char.to_a
107
+ data_row = []
108
+ @fields_info.each_with_index { |field_info, index|
109
+ field_value =
110
+ if nil_mask[index] == '1'
111
+ nil
112
+ else
113
+ case field_info[:data_type]
114
+ when 3 # int, NULL w/o cast
115
+ @raw_file.read(field_info[:max_length]).unpack('I')[0]
116
+ when 5 # float
117
+ @raw_file.read(field_info[:max_length]).unpack('D')[0]
118
+ when 11 # bit
119
+ (@raw_file.read(field_info[:max_length]).unpack('B')[0] == '1') ? true : false
120
+ when 20 # bigint
121
+ @raw_file.read(field_info[:max_length]).unpack('L')[0]
122
+ when 129 # varchar, char
123
+ data_len = @raw_file.read(4).unpack('I')[0]
124
+ @raw_file.read(data_len).encode({invalid: :replace, undef: :replace})
125
+ when 130 # nvarchar, nchar
126
+ data_len = @raw_file.read(4).unpack('I')[0]
127
+ @raw_file.read(data_len*2).encode(DEFAULT_ENCODING, DEFAULT_RAW_FILE_ENCODING, {invalid: :replace, undef: :replace})
128
+ when 131 # numeric, decimal
129
+ # precision, scale, sign, value
130
+ values_arr = @raw_file.read(field_info[:max_length]).unpack('C3Q')
131
+ "#{('-' if values_arr[2] == 0)}#{values_arr[3].to_s.insert((values_arr[3].to_s.length-values_arr[1]), '.')}"
132
+ when 135 # datetime
133
+ values_arr = @raw_file.read(field_info[:max_length]).unpack('S6I')
134
+ Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime('%F %T.%L')
135
+ when 304 # datetime2
136
+ values_arr = @raw_file.read(field_info[:max_length]).unpack('S6I')
137
+ Time.local(values_arr[0], values_arr[1], values_arr[2], values_arr[3], values_arr[4], values_arr[5], (values_arr[6].to_f/1000.0)).strftime("%F %T.%#{field_info[:scale]}N")
138
+ else
139
+ raise UnknownDataTypeError, "Unknown data type #{field_info[:data_type]}"
140
+ end
141
+ end
142
+ data_row << [ field_info[:name], field_value ]
143
+ }
144
+ rescue
145
+ raise
146
+ else
147
+ data_row.to_h
148
+ end
149
+ end
150
+
151
+ # get metadata (header) of raw file
152
+ #
153
+ # @return [Array] Array of hash of fields metadata
154
+ def header
155
+ @fields_info
156
+ end
157
+
158
+ # get version of raw file
159
+ def version
160
+ @version
161
+ end
162
+ end
@@ -0,0 +1,17 @@
1
+ require 'rubygems'
2
+
3
+ require "#{File.dirname(__FILE__)}/lib/sql_server_dts_ns"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = 'sql-server-raw-file'
7
+ s.version = SqlServerDts::VERSION
8
+ s.licenses = ['MIT']
9
+ s.summary = 'sql-server-raw-file lib'
10
+ s.description = 'Converts SQL Server Integration Services Raw File source to hash.'
11
+ s.authors = ['Dmitriy Mullo']
12
+ s.email = ['d.a.mullo1981@gmail.com']
13
+ s.homepage = 'https://github.com/dim11981/sql-server-raw-file'
14
+ s.platform = Gem::Platform::RUBY
15
+ s.files = Dir['*.md']+Dir['sql-server-raw-file.*']+Dir['lib/*.rb']+Dir['test/*.rb']+Dir['fixtures/*']
16
+ s.require_path = 'lib'
17
+ end
@@ -0,0 +1,44 @@
1
+ # coding: utf-8
2
+ require 'csv'
3
+ require 'json'
4
+
5
+ require "#{File.dirname(__FILE__)}/../lib/sql_server_raw_file"
6
+
7
+ fixtures_path = "#{File.dirname(__FILE__)}/../fixtures/"
8
+ raw_path = %w(sql09.raw sql10.raw sql11.raw)
9
+
10
+ # current_fixture_index = 0
11
+ current_fixture_index = File.read(fixtures_path+'/current_sql_index').to_i
12
+
13
+ # show header of raw file and convert it hash and put it json-file
14
+ File.open(fixtures_path+raw_path[current_fixture_index]) { |raw_io|
15
+ raw_obj = SqlServerDts::RawFile.new(raw_io)
16
+ puts "RAW VERSION: #{raw_obj.version}"
17
+ puts '=== BEGIN HEADER ==='
18
+ puts raw_obj.header
19
+ File.open("#{fixtures_path+File.basename(raw_io.path, '.*')}_header.json",'wt') { |json_io|
20
+ header_json = JSON.pretty_generate(raw_obj.header)
21
+ json_io.write(header_json)
22
+ }
23
+ puts '=== END HEADER ==='
24
+ }
25
+
26
+ # show data of raw file in rows table and convert it hash and put it csv-file
27
+ SqlServerDts::RawFile.new(fixtures_path+raw_path[current_fixture_index]) { |raw_obj, raw_io|
28
+ puts '=== BEGIN DATA ==='
29
+ i=0
30
+ fields_arr = []
31
+ raw_obj.header.each { |field_info| fields_arr << field_info[:name] }
32
+ puts fields_arr.join("\t")
33
+ CSV.open("#{fixtures_path+File.basename(raw_io.path, '.*')}_data.csv", 'wb', {headers: fields_arr, write_headers: true, col_sep: "\t"}) { |csv|
34
+ until raw_io.eof?
35
+ values_arr = []
36
+ raw_obj.read_line.each_value { |v| values_arr << v }
37
+ csv << values_arr
38
+ puts values_arr.join("\t")
39
+ i+=1
40
+ end
41
+ }
42
+ puts '=== END DATA ==='
43
+ puts "total: #{i}"
44
+ }
metadata ADDED
@@ -0,0 +1,63 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sql-server-raw-file
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Dmitriy Mullo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-01-14 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Converts SQL Server Integration Services Raw File source to hash.
14
+ email:
15
+ - d.a.mullo1981@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - CHANGELOG.md
21
+ - LICENSE.md
22
+ - README.md
23
+ - fixtures/current_sql_index
24
+ - fixtures/select-fixture-data-SQL_Server_90.sql
25
+ - fixtures/select-fixture-data.sql
26
+ - fixtures/sql09.raw
27
+ - fixtures/sql09_data.csv
28
+ - fixtures/sql09_header.json
29
+ - fixtures/sql10.raw
30
+ - fixtures/sql10_data.csv
31
+ - fixtures/sql10_header.json
32
+ - fixtures/sql11.raw
33
+ - fixtures/sql11_data.csv
34
+ - fixtures/sql11_header.json
35
+ - lib/sql_server_dts_ns.rb
36
+ - lib/sql_server_raw_file.rb
37
+ - sql-server-raw-file.gemspec
38
+ - test/test_sql_data_and_header.rb
39
+ homepage: https://github.com/dim11981/sql-server-raw-file
40
+ licenses:
41
+ - MIT
42
+ metadata: {}
43
+ post_install_message:
44
+ rdoc_options: []
45
+ require_paths:
46
+ - lib
47
+ required_ruby_version: !ruby/object:Gem::Requirement
48
+ requirements:
49
+ - - ">="
50
+ - !ruby/object:Gem::Version
51
+ version: '0'
52
+ required_rubygems_version: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ">="
55
+ - !ruby/object:Gem::Version
56
+ version: '0'
57
+ requirements: []
58
+ rubyforge_project:
59
+ rubygems_version: 2.4.5.1
60
+ signing_key:
61
+ specification_version: 4
62
+ summary: sql-server-raw-file lib
63
+ test_files: []