so2db 0.1.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +8 -6
- data/Gemfile.lock +31 -26
- data/lib/so2db/migrations.rb +29 -10
- data/lib/so2db/models.rb +17 -8
- data/test/test_formatter.rb +4 -4
- data/test/test_models.rb +4 -4
- data/test/test_so2db.rb +3 -3
- data/test/test_so2pg.rb +4 -4
- metadata +29 -42
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9d9d1ba6e24e1b8dc2b5dac9fea8976d9133b643
|
4
|
+
data.tar.gz: 3b11ff556f95761f0dabc2173ac794e991546857
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9909e3d83b384e3869f787cbcf448c2b0e035c50636573ee34ac0d428581aff452d0fe5751503b5635883e8cc2008494f29854ce7f4907c27698126e71568068
|
7
|
+
data.tar.gz: 8898484f96b444033fd746f4bfbaf249f85d8e0627a2c8dd1df8a8127eb4c308f9c379ba5b4ad53873e513ee6f50a1a7d5f2ed403efcf484acad0b25583f1cf3
|
data/Gemfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
source 'http://rubygems.org'
|
2
2
|
|
3
|
-
gem 'nokogiri'
|
4
|
-
gem 'activerecord'
|
5
|
-
gem 'foreigner'
|
6
|
-
gem 'pg'
|
7
|
-
gem 'rake'
|
8
|
-
gem 'mocha'
|
3
|
+
gem 'nokogiri', '1.5.6'
|
4
|
+
gem 'activerecord', '3.2.8'
|
5
|
+
gem 'foreigner', '1.2.1'
|
6
|
+
gem 'pg', '0.14.0'
|
7
|
+
gem 'rake', '0.9.3'
|
8
|
+
gem 'mocha', '1.1.0', :require => false
|
9
|
+
gem 'fakefs', '0.4.0'
|
10
|
+
gem 'minitest'
|
data/Gemfile.lock
CHANGED
@@ -1,37 +1,42 @@
|
|
1
1
|
GEM
|
2
2
|
remote: http://rubygems.org/
|
3
3
|
specs:
|
4
|
-
activemodel (3.
|
5
|
-
activesupport (= 3.
|
6
|
-
builder (~>
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
builder (
|
17
|
-
|
4
|
+
activemodel (3.2.8)
|
5
|
+
activesupport (= 3.2.8)
|
6
|
+
builder (~> 3.0.0)
|
7
|
+
activerecord (3.2.8)
|
8
|
+
activemodel (= 3.2.8)
|
9
|
+
activesupport (= 3.2.8)
|
10
|
+
arel (~> 3.0.2)
|
11
|
+
tzinfo (~> 0.3.29)
|
12
|
+
activesupport (3.2.8)
|
13
|
+
i18n (~> 0.6)
|
14
|
+
multi_json (~> 1.0)
|
15
|
+
arel (3.0.3)
|
16
|
+
builder (3.0.4)
|
17
|
+
fakefs (0.4.0)
|
18
|
+
foreigner (1.2.1)
|
18
19
|
activerecord (>= 3.0.0)
|
19
|
-
i18n (0.
|
20
|
-
metaclass (0.0.
|
21
|
-
|
20
|
+
i18n (0.7.0)
|
21
|
+
metaclass (0.0.4)
|
22
|
+
minitest (5.4.0)
|
23
|
+
mocha (1.1.0)
|
22
24
|
metaclass (~> 0.0.1)
|
23
|
-
|
25
|
+
multi_json (1.11.1)
|
26
|
+
nokogiri (1.5.6)
|
24
27
|
pg (0.14.0)
|
25
|
-
rake (0.9.
|
26
|
-
tzinfo (0.3.
|
28
|
+
rake (0.9.3)
|
29
|
+
tzinfo (0.3.44)
|
27
30
|
|
28
31
|
PLATFORMS
|
29
32
|
ruby
|
30
33
|
|
31
34
|
DEPENDENCIES
|
32
|
-
activerecord
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
35
|
+
activerecord (= 3.2.8)
|
36
|
+
fakefs (= 0.4.0)
|
37
|
+
foreigner (= 1.2.1)
|
38
|
+
minitest
|
39
|
+
mocha (= 1.1.0)
|
40
|
+
nokogiri (= 1.5.6)
|
41
|
+
pg (= 0.14.0)
|
42
|
+
rake (= 0.9.3)
|
data/lib/so2db/migrations.rb
CHANGED
@@ -33,6 +33,8 @@ module SO2DB
|
|
33
33
|
begin
|
34
34
|
AddForeignKeyMigration.new(from_table, to_table, options).up
|
35
35
|
rescue
|
36
|
+
puts $!.inspect
|
37
|
+
#puts $@
|
36
38
|
s = "Error creating foreign key from #{from_table} to #{to_table}"
|
37
39
|
s << " on column #{options[:column]}" if options.has_key? :column
|
38
40
|
puts s
|
@@ -62,6 +64,8 @@ module SO2DB
|
|
62
64
|
t.integer :user_id
|
63
65
|
t.string :name, :limit => 50
|
64
66
|
t.timestamp :date
|
67
|
+
t.integer :class
|
68
|
+
t.boolean :tag_based
|
65
69
|
end
|
66
70
|
|
67
71
|
create_table :comments do |t|
|
@@ -82,14 +86,14 @@ module SO2DB
|
|
82
86
|
t.integer :view_count
|
83
87
|
t.text :body
|
84
88
|
t.integer :owner_user_id
|
85
|
-
t.
|
89
|
+
t.text :owner_display_name
|
86
90
|
t.integer :last_editor_user_id
|
87
91
|
t.string :last_editor_display_name, :limit => 40
|
88
92
|
t.timestamp :last_edit_date
|
89
93
|
t.timestamp :last_activity_date
|
90
94
|
t.timestamp :community_owned_date
|
91
95
|
t.timestamp :closed_date
|
92
|
-
t.
|
96
|
+
t.text :title
|
93
97
|
t.string :tags, :limit => 150
|
94
98
|
t.integer :answer_count
|
95
99
|
t.integer :comment_count
|
@@ -103,7 +107,7 @@ module SO2DB
|
|
103
107
|
t.timestamp :creation_date
|
104
108
|
t.integer :user_id
|
105
109
|
t.string :user_display_name, :limit => 40
|
106
|
-
t.
|
110
|
+
t.text :comment
|
107
111
|
t.text :text
|
108
112
|
t.integer :close_reason_id
|
109
113
|
end
|
@@ -111,7 +115,7 @@ module SO2DB
|
|
111
115
|
create_table :users do |t|
|
112
116
|
t.integer :reputation
|
113
117
|
t.timestamp :creation_date
|
114
|
-
t.
|
118
|
+
t.text :display_name
|
115
119
|
t.string :email_hash, :limit => 32
|
116
120
|
t.timestamp :last_access_date
|
117
121
|
t.string :website_url, :limit => 300
|
@@ -121,10 +125,11 @@ module SO2DB
|
|
121
125
|
t.integer :views
|
122
126
|
t.integer :up_votes
|
123
127
|
t.integer :down_votes
|
128
|
+
t.integer :account_id
|
124
129
|
end
|
125
130
|
|
126
131
|
create_table :votes do |t|
|
127
|
-
t.primary_key :id
|
132
|
+
#t.primary_key :id
|
128
133
|
t.integer :post_id
|
129
134
|
t.integer :vote_type_id
|
130
135
|
t.timestamp :creation_date
|
@@ -132,10 +137,24 @@ module SO2DB
|
|
132
137
|
t.integer :bounty_amount
|
133
138
|
end
|
134
139
|
|
140
|
+
create_table :post_links do |t|
|
141
|
+
t.date :creation_date
|
142
|
+
t.integer :post_id
|
143
|
+
t.integer :related_post_id
|
144
|
+
t.integer :link_type_id
|
145
|
+
end
|
146
|
+
|
147
|
+
create_table :tags do |t|
|
148
|
+
t.text :tag_name
|
149
|
+
t.integer :count
|
150
|
+
t.integer :excerpt_post_id
|
151
|
+
t.integer :wiki_post_id
|
152
|
+
end
|
135
153
|
end
|
136
154
|
|
155
|
+
|
137
156
|
def down
|
138
|
-
[:votes, :badges, :comments, :post_history, :posts, :users].each do |t|
|
157
|
+
[:votes, :badges, :comments, :post_history, :posts, :users, :post_links, :tags].each do |t|
|
139
158
|
drop_table t
|
140
159
|
end
|
141
160
|
end
|
@@ -179,7 +198,7 @@ module SO2DB
|
|
179
198
|
|
180
199
|
def create_post_types
|
181
200
|
create_table :post_types do |t|
|
182
|
-
t.primary_key :id
|
201
|
+
#t.primary_key :id
|
183
202
|
t.string :type_name, :limit => 24
|
184
203
|
end
|
185
204
|
|
@@ -201,7 +220,7 @@ module SO2DB
|
|
201
220
|
|
202
221
|
def create_post_history_types
|
203
222
|
create_table :post_history_types do |t|
|
204
|
-
t.primary_key :id
|
223
|
+
#t.primary_key :id
|
205
224
|
t.string :name, :limit => 50
|
206
225
|
end
|
207
226
|
|
@@ -245,7 +264,7 @@ module SO2DB
|
|
245
264
|
|
246
265
|
def create_close_reasons
|
247
266
|
create_table :close_reasons do |t|
|
248
|
-
t.primary_key :id
|
267
|
+
#t.primary_key :id
|
249
268
|
t.string :name, :limit => 50
|
250
269
|
end
|
251
270
|
|
@@ -266,7 +285,7 @@ module SO2DB
|
|
266
285
|
|
267
286
|
def create_vote_types
|
268
287
|
create_table :vote_types do |t|
|
269
|
-
t.primary_key :id
|
288
|
+
#t.primary_key :id
|
270
289
|
t.string :name, :limit => 50
|
271
290
|
end
|
272
291
|
|
data/lib/so2db/models.rb
CHANGED
@@ -25,7 +25,7 @@ module SO2DB::Models
|
|
25
25
|
|
26
26
|
class Badge < ActiveRecord::Base
|
27
27
|
def self.exported_fields
|
28
|
-
return [ :id, :user_id, :name, :date ]
|
28
|
+
return [ :id, :user_id, :name, :date, :class, :tag_based ]
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
@@ -61,7 +61,7 @@ module SO2DB::Models
|
|
61
61
|
def self.exported_fields
|
62
62
|
return [ :id, :reputation, :creation_date, :display_name, :email_hash,
|
63
63
|
:last_access_date, :website_url, :location, :age, :about_me,
|
64
|
-
:views, :up_votes, :down_votes ]
|
64
|
+
:views, :up_votes, :down_votes, :account_id ]
|
65
65
|
end
|
66
66
|
end
|
67
67
|
|
@@ -72,6 +72,18 @@ module SO2DB::Models
|
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
75
|
+
class PostLink < ActiveRecord::Base
|
76
|
+
def self.exported_fields
|
77
|
+
return [ :id, :creation_date, :post_id, :related_post_id, :link_type_id ]
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
class Tag < ActiveRecord::Base
|
82
|
+
def self.exported_fields
|
83
|
+
return [ :id, :tag_name, :count, :excerpt_post_id, :wiki_post_id ]
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
75
87
|
class PostType < ActiveRecord::Base
|
76
88
|
end
|
77
89
|
|
@@ -87,20 +99,17 @@ module SO2DB::Models
|
|
87
99
|
# Infrastructure. Do not call this from your code.
|
88
100
|
class Lookup
|
89
101
|
|
90
|
-
@@map = { "badges" => :Badge, "comments" => :Comment,
|
102
|
+
@@map = { "badges" => :Badge, "comments" => :Comment,
|
91
103
|
"posthistory" => :PostHistory, "posts" => :Post, "users" => :User,
|
92
|
-
"votes" => :Vote }
|
104
|
+
"votes" => :Vote, "postlinks" => :PostLink, "tags" => :Tag }
|
93
105
|
|
94
106
|
def self.find_class(file_name)
|
95
|
-
Object.const_get("SO2DB
|
96
|
-
.const_get(@@map[file_name].to_s)
|
107
|
+
Object.const_get("SO2DB::Models::#{@@map[file_name].to_s}")
|
97
108
|
end
|
98
109
|
|
99
110
|
def self.get_required_attrs(file_name)
|
100
111
|
raw = find_class(file_name).send :exported_fields
|
101
112
|
return raw.map {|f| f.to_s.camelize.sub(/Guid/, 'GUID')}
|
102
113
|
end
|
103
|
-
|
104
114
|
end
|
105
|
-
|
106
115
|
end
|
data/test/test_formatter.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
require '
|
2
|
-
require 'mocha'
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'mocha/mini_test'
|
3
3
|
require 'so2db'
|
4
4
|
|
5
|
-
class FormatterTest < Test
|
5
|
+
class FormatterTest < Minitest::Test
|
6
6
|
include Rake::DSL
|
7
7
|
|
8
8
|
def setup
|
@@ -61,7 +61,7 @@ class FormatterTest < Test::Unit::TestCase
|
|
61
61
|
end
|
62
62
|
|
63
63
|
def test_value_str
|
64
|
-
assert_equal "badges(date,id,name,user_id)", @formatter.value_str
|
64
|
+
assert_equal "badges(class,date,id,name,tag_based,user_id)", @formatter.value_str
|
65
65
|
end
|
66
66
|
|
67
67
|
def create_node_stub(name, node_type)
|
data/test/test_models.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
require '
|
1
|
+
require 'minitest/autorun'
|
2
2
|
require 'so2db'
|
3
3
|
|
4
|
-
class LookupTest < Test
|
4
|
+
class LookupTest < Minitest::Test
|
5
5
|
include Rake::DSL
|
6
6
|
|
7
7
|
def test_lookup_badges
|
@@ -10,11 +10,11 @@ class LookupTest < Test::Unit::TestCase
|
|
10
10
|
|
11
11
|
def test_required_attrs_badges
|
12
12
|
attrs = SO2DB::Models::Lookup::get_required_attrs("badges")
|
13
|
-
assert_equal [ "Id", "UserId", "Name", "Date" ], attrs
|
13
|
+
assert_equal [ "Id", "UserId", "Name", "Date", "Class", "TagBased" ], attrs
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_guid_capitalization
|
17
17
|
attrs = SO2DB::Models::Lookup::get_required_attrs("posthistory")
|
18
|
-
|
18
|
+
assert(attrs.include? 'RevisionGUID')
|
19
19
|
end
|
20
20
|
end
|
data/test/test_so2db.rb
CHANGED
data/test/test_so2pg.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
require '
|
2
|
-
require 'mocha'
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'mocha/mini_test'
|
3
3
|
require 'so2pg'
|
4
4
|
|
5
|
-
class PgImporterTest < Test
|
5
|
+
class PgImporterTest < Minitest::Test
|
6
6
|
|
7
7
|
def setup
|
8
8
|
@importer = PgImporter.new(true, true, { :database => "dbname", :dir => "dir" })
|
@@ -57,7 +57,7 @@ class PgImporterTest < Test::Unit::TestCase
|
|
57
57
|
|
58
58
|
end
|
59
59
|
|
60
|
-
class PgOptionsParserTest < Test
|
60
|
+
class PgOptionsParserTest < Minitest::Test
|
61
61
|
|
62
62
|
def test_all_options
|
63
63
|
host = 'localhost'
|
metadata
CHANGED
@@ -1,132 +1,120 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: so2db
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.1
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Chad Taylor
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2015-06-29 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: activerecord
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: foreigner
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: nokogiri
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - ">="
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - ">="
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: pg
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - ">="
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: '0'
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - ">="
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: '0'
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
70
|
name: rake
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
|
-
- -
|
73
|
+
- - ">="
|
84
74
|
- !ruby/object:Gem::Version
|
85
75
|
version: '0'
|
86
76
|
type: :development
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
|
-
- -
|
80
|
+
- - ">="
|
92
81
|
- !ruby/object:Gem::Version
|
93
82
|
version: '0'
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
84
|
name: mocha
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
|
-
- -
|
87
|
+
- - ">="
|
100
88
|
- !ruby/object:Gem::Version
|
101
89
|
version: '0'
|
102
90
|
type: :development
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
|
-
- -
|
94
|
+
- - ">="
|
108
95
|
- !ruby/object:Gem::Version
|
109
96
|
version: '0'
|
110
|
-
description:
|
111
|
-
|
97
|
+
description: |2
|
98
|
+
SO2DB provides an API for building StackOverflow data dump importers. It
|
99
|
+
ships with a PostgreSQL implementation and binary.
|
112
100
|
email: caepo3@gmail.com
|
113
101
|
executables:
|
114
102
|
- so2pg
|
115
103
|
extensions: []
|
116
104
|
extra_rdoc_files: []
|
117
105
|
files:
|
106
|
+
- Gemfile
|
107
|
+
- Gemfile.lock
|
108
|
+
- MIT-LICENSE
|
109
|
+
- README.md
|
110
|
+
- Rakefile
|
111
|
+
- bin/so2pg
|
112
|
+
- lib/so2db.rb
|
118
113
|
- lib/so2db/extensions.rb
|
119
114
|
- lib/so2db/formatter.rb
|
120
115
|
- lib/so2db/migrations.rb
|
121
116
|
- lib/so2db/models.rb
|
122
|
-
- lib/so2db.rb
|
123
117
|
- lib/so2pg.rb
|
124
|
-
- bin/so2pg
|
125
|
-
- Gemfile
|
126
|
-
- Gemfile.lock
|
127
|
-
- MIT-LICENSE
|
128
|
-
- Rakefile
|
129
|
-
- README.md
|
130
118
|
- test/test_formatter.rb
|
131
119
|
- test/test_models.rb
|
132
120
|
- test/test_so2db.rb
|
@@ -134,27 +122,26 @@ files:
|
|
134
122
|
homepage: https://github.com/tessellator/so2db
|
135
123
|
licenses:
|
136
124
|
- MIT
|
125
|
+
metadata: {}
|
137
126
|
post_install_message:
|
138
127
|
rdoc_options: []
|
139
128
|
require_paths:
|
140
129
|
- lib
|
141
130
|
required_ruby_version: !ruby/object:Gem::Requirement
|
142
|
-
none: false
|
143
131
|
requirements:
|
144
|
-
- -
|
132
|
+
- - ">="
|
145
133
|
- !ruby/object:Gem::Version
|
146
134
|
version: '0'
|
147
135
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
148
|
-
none: false
|
149
136
|
requirements:
|
150
|
-
- -
|
137
|
+
- - ">="
|
151
138
|
- !ruby/object:Gem::Version
|
152
139
|
version: '0'
|
153
140
|
requirements: []
|
154
141
|
rubyforge_project:
|
155
|
-
rubygems_version:
|
142
|
+
rubygems_version: 2.4.1
|
156
143
|
signing_key:
|
157
|
-
specification_version:
|
144
|
+
specification_version: 4
|
158
145
|
summary: StackOverflow Data Dump Import Utilities
|
159
146
|
test_files:
|
160
147
|
- test/test_formatter.rb
|