openc_bot 0.0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/.travis.yml +8 -0
- data/CHANGELOG.md +2 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +253 -0
- data/Rakefile +14 -0
- data/bin/openc_bot +13 -0
- data/create_bot.sh +30 -0
- data/create_company_bot.sh +16 -0
- data/create_simple_licence_bot.sh +31 -0
- data/db/.gitkeep +0 -0
- data/examples/basic/.gitignore +3 -0
- data/examples/basic/Gemfile +7 -0
- data/examples/basic/config.yml +21 -0
- data/examples/basic/lib/basic.rb +88 -0
- data/examples/basic_with_proxy/Gemfile +7 -0
- data/examples/basic_with_proxy/config.yml +21 -0
- data/examples/basic_with_proxy/lib/basic_with_proxy.rb +103 -0
- data/examples/bot_with_simple_iterator/Gemfile +6 -0
- data/examples/bot_with_simple_iterator/config.yml +21 -0
- data/examples/bot_with_simple_iterator/lib/bot_with_simple_iterator.rb +112 -0
- data/examples/company_fetchers/basic.rb +49 -0
- data/lib/monkey_patches/mechanize.rb +53 -0
- data/lib/openc_bot.rb +89 -0
- data/lib/openc_bot/bot_data_validator.rb +18 -0
- data/lib/openc_bot/company_fetcher_bot.rb +40 -0
- data/lib/openc_bot/exceptions.rb +17 -0
- data/lib/openc_bot/helpers/_csv.rb +10 -0
- data/lib/openc_bot/helpers/alpha_search.rb +73 -0
- data/lib/openc_bot/helpers/dates.rb +33 -0
- data/lib/openc_bot/helpers/html.rb +8 -0
- data/lib/openc_bot/helpers/incremental_search.rb +106 -0
- data/lib/openc_bot/helpers/register_methods.rb +205 -0
- data/lib/openc_bot/helpers/text.rb +18 -0
- data/lib/openc_bot/incrementers.rb +2 -0
- data/lib/openc_bot/incrementers/base.rb +214 -0
- data/lib/openc_bot/incrementers/common.rb +47 -0
- data/lib/openc_bot/tasks.rb +385 -0
- data/lib/openc_bot/templates/README.md +35 -0
- data/lib/openc_bot/templates/bin/export_data +28 -0
- data/lib/openc_bot/templates/bin/fetch_data +23 -0
- data/lib/openc_bot/templates/bin/verify_data +1 -0
- data/lib/openc_bot/templates/config.yml +21 -0
- data/lib/openc_bot/templates/lib/bot.rb +43 -0
- data/lib/openc_bot/templates/lib/company_fetcher_bot.rb +95 -0
- data/lib/openc_bot/templates/lib/simple_bot.rb +67 -0
- data/lib/openc_bot/templates/spec/bot_spec.rb +11 -0
- data/lib/openc_bot/templates/spec/simple_bot_spec.rb +11 -0
- data/lib/openc_bot/templates/spec/spec_helper.rb +13 -0
- data/lib/openc_bot/version.rb +3 -0
- data/lib/simple_openc_bot.rb +289 -0
- data/openc_bot.gemspec +35 -0
- data/schemas/company-schema.json +112 -0
- data/schemas/includes/address.json +23 -0
- data/schemas/includes/base-statement.json +27 -0
- data/schemas/includes/company.json +14 -0
- data/schemas/includes/filing.json +20 -0
- data/schemas/includes/license-data.json +27 -0
- data/schemas/includes/officer.json +14 -0
- data/schemas/includes/previous_name.json +11 -0
- data/schemas/includes/share-parcel-data.json +67 -0
- data/schemas/includes/share-parcel.json +60 -0
- data/schemas/includes/subsidiary-relationship-data.json +52 -0
- data/schemas/includes/total-shares.json +10 -0
- data/schemas/licence-schema.json +21 -0
- data/schemas/share-parcel-schema.json +21 -0
- data/schemas/subsidiary-relationship-schema.json +19 -0
- data/spec/dummy_classes/foo_bot.rb +4 -0
- data/spec/lib/bot_data_validator_spec.rb +69 -0
- data/spec/lib/company_fetcher_bot_spec.rb +93 -0
- data/spec/lib/exceptions_spec.rb +25 -0
- data/spec/lib/helpers/alpha_search_spec.rb +173 -0
- data/spec/lib/helpers/dates_spec.rb +65 -0
- data/spec/lib/helpers/incremental_search_spec.rb +471 -0
- data/spec/lib/helpers/register_methods_spec.rb +558 -0
- data/spec/lib/helpers/text_spec.rb +50 -0
- data/spec/lib/openc_bot/db/.gitkeep +0 -0
- data/spec/lib/openc_bot/incrementers/common_spec.rb +83 -0
- data/spec/lib/openc_bot_spec.rb +116 -0
- data/spec/schemas/company-schema_spec.rb +676 -0
- data/spec/simple_openc_bot_spec.rb +302 -0
- data/spec/spec_helper.rb +19 -0
- metadata +300 -0
@@ -0,0 +1,65 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative '../../spec_helper'
|
3
|
+
require 'openc_bot/helpers/dates'
|
4
|
+
|
5
|
+
describe OpencBot::Helpers::Dates do
|
6
|
+
describe "#normalise_us_date" do
|
7
|
+
it "should return nil if blank" do
|
8
|
+
OpencBot::Helpers::Dates.normalise_us_date(nil).should be_nil
|
9
|
+
OpencBot::Helpers::Dates.normalise_us_date('').should be_nil
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should convert date to string" do
|
13
|
+
date = Date.today
|
14
|
+
OpencBot::Helpers::Dates.normalise_us_date(date).should == date.to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should convert US date if in slash format" do
|
18
|
+
OpencBot::Helpers::Dates.normalise_us_date('01/04/2006').to_s.should == '2006-01-04'
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should convert two digit year " do
|
22
|
+
OpencBot::Helpers::Dates.normalise_us_date('23-Aug-10').to_s.should == '2010-08-23'
|
23
|
+
OpencBot::Helpers::Dates.normalise_us_date('23-Aug-98').to_s.should == '1998-08-23'
|
24
|
+
OpencBot::Helpers::Dates.normalise_us_date('05/Oct/10').to_s.should == '2010-10-05'
|
25
|
+
OpencBot::Helpers::Dates.normalise_us_date('05/10/10').to_s.should == '2010-05-10'
|
26
|
+
OpencBot::Helpers::Dates.normalise_us_date('5/6/10').to_s.should == '2010-05-06'
|
27
|
+
OpencBot::Helpers::Dates.normalise_us_date('5/6/31').to_s.should == '1931-05-06'
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should not convert date if not in slash format" do
|
31
|
+
OpencBot::Helpers::Dates.normalise_us_date('2006-01-04').to_s.should == '2006-01-04'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "when normalising uk date" do
|
36
|
+
it "should return nil if blank" do
|
37
|
+
OpencBot::Helpers::Dates.normalise_uk_date(nil).should be_nil
|
38
|
+
OpencBot::Helpers::Dates.normalise_uk_date('').should be_nil
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should convert date to string" do
|
42
|
+
date = Date.today - 30
|
43
|
+
OpencBot::Helpers::Dates.normalise_uk_date(date).should == date.to_s
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should convert UK date if in slash format" do
|
47
|
+
OpencBot::Helpers::Dates.normalise_uk_date('01/04/2006').to_s.should == '2006-04-01'
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should convert UK date if in dot format" do
|
51
|
+
OpencBot::Helpers::Dates.normalise_uk_date('01.04.2006').to_s.should == '2006-04-01'
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should convert two digit year " do
|
55
|
+
OpencBot::Helpers::Dates.normalise_uk_date('23-Aug-10').to_s.should == '2010-08-23'
|
56
|
+
OpencBot::Helpers::Dates.normalise_uk_date('23-Aug-98').to_s.should == '1998-08-23'
|
57
|
+
OpencBot::Helpers::Dates.normalise_uk_date('05/Oct/10').to_s.should == '2010-10-05'
|
58
|
+
OpencBot::Helpers::Dates.normalise_uk_date('05/10/10').to_s.should == '2010-10-05'
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should not convert date if not in slash format" do
|
62
|
+
OpencBot::Helpers::Dates.normalise_uk_date('2006-01-04').to_s.should == '2006-01-04'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,471 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative '../../spec_helper'
|
3
|
+
require 'openc_bot'
|
4
|
+
require 'openc_bot/helpers/incremental_search'
|
5
|
+
|
6
|
+
module ModuleThatIncludesIncrementalSearch
|
7
|
+
extend OpencBot
|
8
|
+
extend OpencBot::Helpers::IncrementalSearch
|
9
|
+
PRIMARY_KEY_NAME = :custom_uid
|
10
|
+
end
|
11
|
+
|
12
|
+
module ModuleWithNoCustomPrimaryKey
|
13
|
+
extend OpencBot
|
14
|
+
extend OpencBot::Helpers::IncrementalSearch
|
15
|
+
end
|
16
|
+
|
17
|
+
describe 'a module that includes IncrementalSearch' do
|
18
|
+
|
19
|
+
before do
|
20
|
+
ModuleThatIncludesIncrementalSearch.stub(:sqlite_magic_connection).and_return(test_database_connection)
|
21
|
+
end
|
22
|
+
|
23
|
+
after do
|
24
|
+
remove_test_database
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should have #increment_number method" do
|
28
|
+
ModuleThatIncludesIncrementalSearch.should respond_to(:increment_number)
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "increment_number" do
|
32
|
+
it "should increase integer as number" do
|
33
|
+
ModuleThatIncludesIncrementalSearch.increment_number('234567').should == '234568'
|
34
|
+
ModuleThatIncludesIncrementalSearch.increment_number('999').should == '1000'
|
35
|
+
ModuleThatIncludesIncrementalSearch.increment_number('000123').should == '000124'
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should convert numbers to string" do
|
39
|
+
ModuleThatIncludesIncrementalSearch.increment_number(234567).should == '234568'
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should increase number prefixed by string" do
|
43
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B123456').should == 'B123457'
|
44
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000456').should == 'B000457'
|
45
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-999').should == 'B-1000'
|
46
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-000999').should == 'B-001000'
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should increase number suffixed by string" do
|
50
|
+
ModuleThatIncludesIncrementalSearch.increment_number('123456B').should == '123457B'
|
51
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000456B').should == 'B000457B'
|
52
|
+
ModuleThatIncludesIncrementalSearch.increment_number('999-B').should == '1000-B'
|
53
|
+
ModuleThatIncludesIncrementalSearch.increment_number('000999-B').should == '001000-B'
|
54
|
+
end
|
55
|
+
|
56
|
+
context "and number to increment given as option" do
|
57
|
+
it "should increment by given number" do
|
58
|
+
ModuleThatIncludesIncrementalSearch.increment_number('234567', 4).should == '234571'
|
59
|
+
ModuleThatIncludesIncrementalSearch.increment_number('999', 4).should == '1003'
|
60
|
+
ModuleThatIncludesIncrementalSearch.increment_number('000123', 4).should == '000127'
|
61
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B123456', 3).should == 'B123459'
|
62
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000456', 3).should == 'B000459'
|
63
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-999', 4).should == 'B-1003'
|
64
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-000999', 4).should == 'B-001003'
|
65
|
+
ModuleThatIncludesIncrementalSearch.increment_number('123456B',3).should == '123459B'
|
66
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000456B',3).should == 'B000459B'
|
67
|
+
ModuleThatIncludesIncrementalSearch.increment_number('999-B',4).should == '1003-B'
|
68
|
+
ModuleThatIncludesIncrementalSearch.increment_number('000999-B',4).should == '001003-B'
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
|
73
|
+
context "and negative number to increment given as option" do
|
74
|
+
it "should increment by given number" do
|
75
|
+
ModuleThatIncludesIncrementalSearch.increment_number('234567', -3).should == '234564'
|
76
|
+
ModuleThatIncludesIncrementalSearch.increment_number('1002', -3).should == '999'
|
77
|
+
ModuleThatIncludesIncrementalSearch.increment_number('000127', -3).should == '000124'
|
78
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B123459', -3).should == 'B123456'
|
79
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000459', -3).should == 'B000456'
|
80
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-1003', -4).should == 'B-999'
|
81
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B-000999', -3).should == 'B-000996'
|
82
|
+
ModuleThatIncludesIncrementalSearch.increment_number('123459B', -3).should == '123456B'
|
83
|
+
ModuleThatIncludesIncrementalSearch.increment_number('B000459B', -3).should == 'B000456B'
|
84
|
+
ModuleThatIncludesIncrementalSearch.increment_number('1003-B', -4).should == '999-B'
|
85
|
+
ModuleThatIncludesIncrementalSearch.increment_number('001003-B', -4).should == '000999-B'
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should have #incremental_search method" do
|
92
|
+
ModuleThatIncludesIncrementalSearch.should respond_to(:incremental_search)
|
93
|
+
end
|
94
|
+
|
95
|
+
describe '#incremental_search' do
|
96
|
+
before do
|
97
|
+
ModuleThatIncludesIncrementalSearch.stub(:create_new_company)
|
98
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 0)
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should iterate with prefixes, incrementing digits' do
|
102
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12345', anything).and_return(:entry_1)
|
103
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12346', anything).and_return(:entry_2)
|
104
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12347', anything).and_return(nil)
|
105
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('12348', anything)
|
106
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('12345')
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'should use increment_number to, er, increment number' do
|
110
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).and_return(:entry_1)
|
111
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('25632', anything).and_return(nil)
|
112
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:increment_number).with('12345').and_return('34567')
|
113
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:increment_number).with('34567').and_return('76543')
|
114
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:increment_number).with('76543').and_return('25632')
|
115
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('12345')
|
116
|
+
end
|
117
|
+
|
118
|
+
it 'should get company details for given company number and subsequent company numbers until nil is returned more than max count' do
|
119
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 2)
|
120
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234568', anything).and_return(:entry_1)
|
121
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234569', anything).and_return(:entry_2)
|
122
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234570', anything).and_return(nil)
|
123
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234571', anything).and_return(nil)
|
124
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234572', anything).and_return(:entry_3)
|
125
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234573', anything).and_return(nil)
|
126
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234574', anything).and_return(nil)
|
127
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234575', anything).and_return(nil)
|
128
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234576', anything)
|
129
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568')
|
130
|
+
end
|
131
|
+
|
132
|
+
it 'should return last good number' do
|
133
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 2)
|
134
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234568', anything).and_return(:entry_1)
|
135
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234569', anything).and_return(:entry_2)
|
136
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234570', anything).and_return(nil)
|
137
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234571', anything).and_return(nil)
|
138
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234572', anything).and_return(:entry_3)
|
139
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234573', anything).and_return(nil)
|
140
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234574', anything).and_return(nil)
|
141
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with('1234575', anything).and_return(nil)
|
142
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568').should == '1234572'
|
143
|
+
end
|
144
|
+
|
145
|
+
it 'should return given number if no successful results' do
|
146
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 2)
|
147
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).and_return(nil)
|
148
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568').should == '1234568'
|
149
|
+
end
|
150
|
+
|
151
|
+
it 'should pass on any options' do
|
152
|
+
pending 'deciding whether this is useful (implementation carried over from openc)'
|
153
|
+
ModuleThatIncludesIncrementalSearch.stub(:update_datum).with(anything, :foo => 'bar')
|
154
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568', :foo => 'bar')
|
155
|
+
end
|
156
|
+
|
157
|
+
context 'and :skip_existing_entries passed in' do
|
158
|
+
before do
|
159
|
+
ModuleThatIncludesIncrementalSearch.stub(:datum_exists?).and_return(false)
|
160
|
+
ModuleThatIncludesIncrementalSearch.stub(:datum_exists?).with('1234569').and_return(true)
|
161
|
+
ModuleThatIncludesIncrementalSearch.stub(:datum_exists?).with('1234571').and_return(true)
|
162
|
+
end
|
163
|
+
|
164
|
+
it "should not update_datum for existing entries" do
|
165
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 2)
|
166
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234568', anything).and_return(:entry_1)
|
167
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234569', anything)
|
168
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234570', anything).and_return(nil)
|
169
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234571', anything)
|
170
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234572', anything).and_return(:entry_3)
|
171
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234573', anything).and_return(nil)
|
172
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234574', anything).and_return(nil)
|
173
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234575', anything).and_return(nil)
|
174
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234576', anything)
|
175
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568', :skip_existing_entries => true)
|
176
|
+
end
|
177
|
+
|
178
|
+
it "should reset error count when existing entry found" do
|
179
|
+
ModuleThatIncludesIncrementalSearch.stub(:max_failed_count => 1)
|
180
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234568', anything).and_return(nil)
|
181
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234569', anything)
|
182
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234570', anything).and_return(nil)
|
183
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234571', anything)
|
184
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234572', anything).and_return(:entry_3)
|
185
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234573', anything).and_return(nil)
|
186
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('1234574', anything).and_return(nil)
|
187
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('1234575', anything)
|
188
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('1234568', :skip_existing_entries => true)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
context 'and offset passed in as option' do
|
193
|
+
it "should get entries beginning from given number adjusted by offset" do
|
194
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12342', anything).and_return(:entry_1)
|
195
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12343', anything).and_return(:entry_2)
|
196
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:update_datum).with('12344', anything).and_return(nil)
|
197
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:update_datum).with('12345', anything)
|
198
|
+
ModuleThatIncludesIncrementalSearch.incremental_search('12345', :offset => -3)
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
describe '#incremental_rewind_count' do
|
204
|
+
it 'should return nil if INCREMENTAL_REWIND_COUNT not set' do
|
205
|
+
ModuleThatIncludesIncrementalSearch.send(:incremental_rewind_count).should be_nil
|
206
|
+
end
|
207
|
+
|
208
|
+
it 'should return value if INCREMENTAL_REWIND_COUNT set' do
|
209
|
+
stub_const("ModuleThatIncludesIncrementalSearch::INCREMENTAL_REWIND_COUNT", 42)
|
210
|
+
ModuleThatIncludesIncrementalSearch.send(:incremental_rewind_count).should == ModuleThatIncludesIncrementalSearch::INCREMENTAL_REWIND_COUNT
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
describe '#max_failed_count' do
|
215
|
+
it 'should return 0 if MAX_FAILED_COUNT not set' do
|
216
|
+
ModuleThatIncludesIncrementalSearch.send(:max_failed_count).should == 10
|
217
|
+
end
|
218
|
+
|
219
|
+
it 'should return value if MAX_FAILED_COUNT set' do
|
220
|
+
stub_const("ModuleThatIncludesIncrementalSearch::MAX_FAILED_COUNT", 42)
|
221
|
+
ModuleThatIncludesIncrementalSearch.send(:max_failed_count).should == ModuleThatIncludesIncrementalSearch::MAX_FAILED_COUNT
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
describe "entity_uid_prefixes" do
|
226
|
+
context 'and no ENTITY_UID_PREFIXES constant' do
|
227
|
+
it "should return array containing nil" do
|
228
|
+
ModuleThatIncludesIncrementalSearch.entity_uid_prefixes.should == [nil]
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
context 'and has ENTITY_UID_PREFIXES constant' do
|
233
|
+
it "should return ENTITY_UID_PREFIXES" do
|
234
|
+
stub_const("ModuleThatIncludesIncrementalSearch::ENTITY_UID_PREFIXES", ['A','X'])
|
235
|
+
ModuleThatIncludesIncrementalSearch.entity_uid_prefixes.should == ModuleThatIncludesIncrementalSearch::ENTITY_UID_PREFIXES
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
describe "entity_uid_suffixes" do
|
241
|
+
context 'and no ENTITY_UID_SUFFIXES constant' do
|
242
|
+
it "should return array containing nil" do
|
243
|
+
ModuleThatIncludesIncrementalSearch.entity_uid_suffixes.should == [nil]
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
context 'and has ENTITY_UID_SUFFIXES constant' do
|
248
|
+
it "should return ENTITY_UID_SUFFIXES" do
|
249
|
+
stub_const("ModuleThatIncludesIncrementalSearch::ENTITY_UID_SUFFIXES", ['A','X'])
|
250
|
+
ModuleThatIncludesIncrementalSearch.entity_uid_suffixes.should == ModuleThatIncludesIncrementalSearch::ENTITY_UID_SUFFIXES
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
describe '#fetch_data_via_incremental_search' do
|
256
|
+
before do
|
257
|
+
@most_recent_companies = ['03456789', 'A12345']
|
258
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uids).and_return(@most_recent_companies)
|
259
|
+
ModuleThatIncludesIncrementalSearch.stub(:incremental_search)
|
260
|
+
end
|
261
|
+
|
262
|
+
it 'should find highest_entry_uids' do
|
263
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:highest_entry_uids)
|
264
|
+
|
265
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
266
|
+
end
|
267
|
+
|
268
|
+
context 'and highest_entry_uids returns nil' do
|
269
|
+
before do
|
270
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uids) # => nil
|
271
|
+
end
|
272
|
+
|
273
|
+
it 'should not do incremental_search' do
|
274
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:incremental_search)
|
275
|
+
|
276
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
277
|
+
end
|
278
|
+
|
279
|
+
it 'should not update cached highest_entry_uid value' do
|
280
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:save_var)
|
281
|
+
|
282
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
context 'and highest_entry_uids returns values' do
|
287
|
+
before do
|
288
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uids).and_return(@most_recent_companies)
|
289
|
+
end
|
290
|
+
|
291
|
+
it 'should do incremental_search starting at each highest company number' do
|
292
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('03456789', {})
|
293
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('A12345', {})
|
294
|
+
|
295
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
296
|
+
end
|
297
|
+
|
298
|
+
it 'should save highest_entry_uids' do
|
299
|
+
ModuleThatIncludesIncrementalSearch.stub(:incremental_search).with('03456789', anything).and_return('0345999')
|
300
|
+
ModuleThatIncludesIncrementalSearch.stub(:incremental_search).with('A12345', anything).and_return('A234567')
|
301
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:save_var).with(:highest_entry_uids, ['0345999','A234567'])
|
302
|
+
|
303
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
304
|
+
end
|
305
|
+
|
306
|
+
context 'and options passed to fetch_data_via_incremental_search' do
|
307
|
+
it "should pass them on to incremental search" do
|
308
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('03456789', :foo => 'bar')
|
309
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('A12345', :foo => 'bar')
|
310
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search(:foo => 'bar')
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
314
|
+
context 'and library has incremental_rewind_count' do
|
315
|
+
before do
|
316
|
+
ModuleThatIncludesIncrementalSearch.stub(:incremental_rewind_count).and_return(42)
|
317
|
+
end
|
318
|
+
|
319
|
+
it "should pass negated version on to incremental search as offset" do
|
320
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('03456789', hash_including(:offset => -42))
|
321
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('A12345', hash_including(:offset => -42))
|
322
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
323
|
+
end
|
324
|
+
|
325
|
+
it "should ask to skip_existing_companies on incremental search by default" do
|
326
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('03456789', hash_including(:skip_existing_entries => true))
|
327
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('A12345', hash_including(:skip_existing_entries => true))
|
328
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search
|
329
|
+
end
|
330
|
+
|
331
|
+
it "should not ask to skip_existing_companies on incremental search if requested not to" do
|
332
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('03456789', hash_including(:skip_existing_entries => false))
|
333
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('A12345', hash_including(:skip_existing_entries => false))
|
334
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search(:skip_existing_entries => false)
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
end
|
339
|
+
|
340
|
+
context 'and highest_entry_uids passed in options' do
|
341
|
+
it "should not find highest_entry_uids" do
|
342
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive(:highest_entry_uids)
|
343
|
+
|
344
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search(:highest_entry_uids => ['1234', '6543'])
|
345
|
+
end
|
346
|
+
|
347
|
+
it 'should do incremental_search starting at provided highest company numbers' do
|
348
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('1234', {})
|
349
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:incremental_search).with('6543', {})
|
350
|
+
|
351
|
+
ModuleThatIncludesIncrementalSearch.fetch_data_via_incremental_search(:highest_entry_uids => ['1234', '6543'])
|
352
|
+
end
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
describe '#highest_entry_uids' do
|
357
|
+
before do
|
358
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uid_result).and_return('553311')
|
359
|
+
ModuleThatIncludesIncrementalSearch.stub(:entity_uid_prefixes).and_return(['H', 'P'])
|
360
|
+
end
|
361
|
+
|
362
|
+
context 'and highest_entry_uids not set in cache' do
|
363
|
+
|
364
|
+
it 'should get highest_entry_uid_result for each prefix' do
|
365
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:highest_entry_uid_result).with(:prefix => 'H').with(:prefix => 'P')
|
366
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids
|
367
|
+
end
|
368
|
+
|
369
|
+
it 'should return highest uid for jurisdiction_code' do
|
370
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:highest_entry_uid_result).with(:prefix => 'H').and_return('H553311')
|
371
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:highest_entry_uid_result).with(:prefix => 'P').and_return('P12345')
|
372
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids.should == ['H553311', 'P12345']
|
373
|
+
end
|
374
|
+
end
|
375
|
+
|
376
|
+
context 'and highest_entry_uid_result returns nil value' do
|
377
|
+
before do
|
378
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uid_result).with(:prefix => 'H').and_return(nil)
|
379
|
+
ModuleThatIncludesIncrementalSearch.stub(:highest_entry_uid_result).with(:prefix => 'P').and_return('P12345')
|
380
|
+
end
|
381
|
+
|
382
|
+
it "should return only nil values" do
|
383
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids.should == ['P12345']
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
context 'and highest_uid in cache' do
|
388
|
+
before do
|
389
|
+
@cached_result = '765432'
|
390
|
+
ModuleThatIncludesIncrementalSearch.save_var('highest_entry_uids', @cached_result)
|
391
|
+
end
|
392
|
+
|
393
|
+
it 'should not search for companies' do
|
394
|
+
ModuleThatIncludesIncrementalSearch.should_not_receive('highest_entry_uid_result')
|
395
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids
|
396
|
+
end
|
397
|
+
|
398
|
+
it 'should return cached highest_entry_uid' do
|
399
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids.should == @cached_result
|
400
|
+
end
|
401
|
+
|
402
|
+
context 'and blank value returned for cached value' do
|
403
|
+
before do
|
404
|
+
ModuleThatIncludesIncrementalSearch.save_var('highest_entry_uids', ['H553311',''])
|
405
|
+
end
|
406
|
+
|
407
|
+
it "should ignore cached value" do
|
408
|
+
ModuleThatIncludesIncrementalSearch.should_receive(:highest_entry_uid_result).with(:prefix => 'H').with(:prefix => 'P')
|
409
|
+
|
410
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uids
|
411
|
+
end
|
412
|
+
end
|
413
|
+
end
|
414
|
+
|
415
|
+
end
|
416
|
+
|
417
|
+
describe '#highest_entry_uid_result' do
|
418
|
+
context "in general" do
|
419
|
+
before do
|
420
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '99999')
|
421
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '5234888')
|
422
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '9234567')
|
423
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => 'A094567')
|
424
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => 'A234567')
|
425
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => 'SL34567')
|
426
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => 'SL34999')
|
427
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => 'H9999')
|
428
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '5234567')
|
429
|
+
end
|
430
|
+
|
431
|
+
context 'and no options passed' do
|
432
|
+
it 'should return highest uid as number for jurisdiction_code' do
|
433
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result.should == '9234567'
|
434
|
+
end
|
435
|
+
end
|
436
|
+
|
437
|
+
context 'and prefix passed in options' do
|
438
|
+
it 'should return highest uid as number for jurisdiction_code' do
|
439
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result(:prefix => 'A').should == 'A234567'
|
440
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result(:prefix => 'SL').should == 'SL34999'
|
441
|
+
end
|
442
|
+
end
|
443
|
+
|
444
|
+
context "and suffix passed in options" do
|
445
|
+
before do
|
446
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '009802V')
|
447
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '001234V')
|
448
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '34567C')
|
449
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '128055C')
|
450
|
+
ModuleThatIncludesIncrementalSearch.save_data([:custom_uid], :custom_uid => '99999999')
|
451
|
+
end
|
452
|
+
|
453
|
+
it 'should return highest company_number for Im with prefix letter' do
|
454
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result(:suffix => 'V').should == '009802V'
|
455
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result(:suffix => 'C').should == '128055C'
|
456
|
+
end
|
457
|
+
end
|
458
|
+
end
|
459
|
+
|
460
|
+
context "and no database created yet" do
|
461
|
+
it "should return 0" do
|
462
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result.should == '0'
|
463
|
+
end
|
464
|
+
|
465
|
+
it "should return prefix plus 0 if prefix given" do
|
466
|
+
ModuleThatIncludesIncrementalSearch.highest_entry_uid_result(:prefix => 'A').should == 'A0'
|
467
|
+
end
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
end
|