stockboy 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.rspec +5 -0
  4. data/.yardopts +7 -0
  5. data/CHANGELOG.md +24 -0
  6. data/Gemfile +12 -0
  7. data/Guardfile +10 -0
  8. data/LICENSE +21 -0
  9. data/README.md +293 -0
  10. data/Rakefile +30 -0
  11. data/lib/stockboy.rb +80 -0
  12. data/lib/stockboy/attribute.rb +11 -0
  13. data/lib/stockboy/attribute_map.rb +74 -0
  14. data/lib/stockboy/candidate_record.rb +130 -0
  15. data/lib/stockboy/configuration.rb +62 -0
  16. data/lib/stockboy/configurator.rb +176 -0
  17. data/lib/stockboy/dsl.rb +68 -0
  18. data/lib/stockboy/exceptions.rb +3 -0
  19. data/lib/stockboy/filter.rb +58 -0
  20. data/lib/stockboy/filter_chain.rb +41 -0
  21. data/lib/stockboy/filters.rb +11 -0
  22. data/lib/stockboy/filters/missing_email.rb +37 -0
  23. data/lib/stockboy/job.rb +241 -0
  24. data/lib/stockboy/mapped_record.rb +59 -0
  25. data/lib/stockboy/provider.rb +238 -0
  26. data/lib/stockboy/providers.rb +11 -0
  27. data/lib/stockboy/providers/file.rb +135 -0
  28. data/lib/stockboy/providers/ftp.rb +205 -0
  29. data/lib/stockboy/providers/http.rb +123 -0
  30. data/lib/stockboy/providers/imap.rb +290 -0
  31. data/lib/stockboy/providers/soap.rb +120 -0
  32. data/lib/stockboy/railtie.rb +28 -0
  33. data/lib/stockboy/reader.rb +59 -0
  34. data/lib/stockboy/readers.rb +11 -0
  35. data/lib/stockboy/readers/csv.rb +115 -0
  36. data/lib/stockboy/readers/fixed_width.rb +121 -0
  37. data/lib/stockboy/readers/spreadsheet.rb +144 -0
  38. data/lib/stockboy/readers/xml.rb +155 -0
  39. data/lib/stockboy/registry.rb +42 -0
  40. data/lib/stockboy/source_record.rb +43 -0
  41. data/lib/stockboy/string_pool.rb +35 -0
  42. data/lib/stockboy/template_file.rb +44 -0
  43. data/lib/stockboy/translations.rb +70 -0
  44. data/lib/stockboy/translations/boolean.rb +58 -0
  45. data/lib/stockboy/translations/date.rb +41 -0
  46. data/lib/stockboy/translations/decimal.rb +33 -0
  47. data/lib/stockboy/translations/default_empty_string.rb +38 -0
  48. data/lib/stockboy/translations/default_false.rb +41 -0
  49. data/lib/stockboy/translations/default_nil.rb +38 -0
  50. data/lib/stockboy/translations/default_true.rb +41 -0
  51. data/lib/stockboy/translations/default_zero.rb +41 -0
  52. data/lib/stockboy/translations/integer.rb +33 -0
  53. data/lib/stockboy/translations/string.rb +33 -0
  54. data/lib/stockboy/translations/time.rb +41 -0
  55. data/lib/stockboy/translations/uk_date.rb +51 -0
  56. data/lib/stockboy/translations/us_date.rb +51 -0
  57. data/lib/stockboy/translator.rb +66 -0
  58. data/lib/stockboy/version.rb +3 -0
  59. data/spec/fixtures/.gitkeep +0 -0
  60. data/spec/fixtures/files/a_garbage.csv +1 -0
  61. data/spec/fixtures/files/test_data-20120101.csv +1 -0
  62. data/spec/fixtures/files/test_data-20120202.csv +1 -0
  63. data/spec/fixtures/files/z_garbage.csv +1 -0
  64. data/spec/fixtures/jobs/test_job.rb +1 -0
  65. data/spec/fixtures/soap/get_list/fault.xml +8 -0
  66. data/spec/fixtures/soap/get_list/success.xml +18 -0
  67. data/spec/fixtures/spreadsheets/test_data.xls +0 -0
  68. data/spec/fixtures/spreadsheets/test_row_options.xls +0 -0
  69. data/spec/fixtures/xml/body.xml +14 -0
  70. data/spec/spec_helper.rb +28 -0
  71. data/spec/stockboy/attribute_map_spec.rb +59 -0
  72. data/spec/stockboy/attribute_spec.rb +11 -0
  73. data/spec/stockboy/candidate_record_spec.rb +150 -0
  74. data/spec/stockboy/configuration_spec.rb +28 -0
  75. data/spec/stockboy/configurator_spec.rb +127 -0
  76. data/spec/stockboy/filter_chain_spec.rb +40 -0
  77. data/spec/stockboy/filter_spec.rb +41 -0
  78. data/spec/stockboy/filters/missing_email_spec.rb +26 -0
  79. data/spec/stockboy/filters_spec.rb +38 -0
  80. data/spec/stockboy/job_spec.rb +238 -0
  81. data/spec/stockboy/mapped_record_spec.rb +30 -0
  82. data/spec/stockboy/provider_spec.rb +34 -0
  83. data/spec/stockboy/providers/file_spec.rb +116 -0
  84. data/spec/stockboy/providers/ftp_spec.rb +143 -0
  85. data/spec/stockboy/providers/http_spec.rb +94 -0
  86. data/spec/stockboy/providers/imap_spec.rb +76 -0
  87. data/spec/stockboy/providers/soap_spec.rb +107 -0
  88. data/spec/stockboy/providers_spec.rb +38 -0
  89. data/spec/stockboy/readers/csv_spec.rb +68 -0
  90. data/spec/stockboy/readers/fixed_width_spec.rb +52 -0
  91. data/spec/stockboy/readers/spreadsheet_spec.rb +121 -0
  92. data/spec/stockboy/readers/xml_spec.rb +94 -0
  93. data/spec/stockboy/readers_spec.rb +30 -0
  94. data/spec/stockboy/source_record_spec.rb +19 -0
  95. data/spec/stockboy/template_file_spec.rb +30 -0
  96. data/spec/stockboy/translations/boolean_spec.rb +48 -0
  97. data/spec/stockboy/translations/date_spec.rb +38 -0
  98. data/spec/stockboy/translations/decimal_spec.rb +23 -0
  99. data/spec/stockboy/translations/default_empty_string_spec.rb +32 -0
  100. data/spec/stockboy/translations/default_false_spec.rb +25 -0
  101. data/spec/stockboy/translations/default_nil_spec.rb +32 -0
  102. data/spec/stockboy/translations/default_true_spec.rb +25 -0
  103. data/spec/stockboy/translations/default_zero_spec.rb +32 -0
  104. data/spec/stockboy/translations/integer_spec.rb +22 -0
  105. data/spec/stockboy/translations/string_spec.rb +22 -0
  106. data/spec/stockboy/translations/time_spec.rb +27 -0
  107. data/spec/stockboy/translations/uk_date_spec.rb +37 -0
  108. data/spec/stockboy/translations/us_date_spec.rb +37 -0
  109. data/spec/stockboy/translations_spec.rb +55 -0
  110. data/spec/stockboy/translator_spec.rb +27 -0
  111. data/stockboy.gemspec +32 -0
  112. metadata +305 -0
@@ -0,0 +1,38 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/providers'
3
+
4
+ module Stockboy
5
+ describe Providers do
6
+
7
+ let(:provider) { double("provider") }
8
+
9
+ describe ".register" do
10
+ it "registers a key and class" do
11
+ Providers.register(:snailmail, provider).should === provider
12
+ end
13
+ end
14
+
15
+ describe ".find" do
16
+ it "returns a provider class" do
17
+ Providers.register(:snailmail, provider)
18
+ Providers.find(:snailmail).should === provider
19
+ end
20
+ end
21
+
22
+ describe ".[]" do
23
+ it "returns a provider class" do
24
+ Providers.register(:snailmail, provider)
25
+ Providers[:snailmail].should === provider
26
+ end
27
+ end
28
+
29
+ describe ".all" do
30
+ it "returns all registered providers" do
31
+ Providers.register(:snailmail, provider)
32
+ Providers.register(:slugmail, provider)
33
+ Providers.all.should include(snailmail: provider, slugmail: provider)
34
+ end
35
+ end
36
+
37
+ end
38
+ end
@@ -0,0 +1,68 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/csv'
3
+
4
+ module Stockboy
5
+ describe Readers::CSV do
6
+
7
+ subject(:reader) { Stockboy::Readers::CSV.new }
8
+
9
+ describe "default options" do
10
+ its(:row_sep) { should be_nil }
11
+ its(:col_sep) { should be_nil }
12
+ its(:quote_char) { should be_nil }
13
+ its(:headers) { should be_true }
14
+ its(:skip_header_rows) { should == 0 }
15
+ its(:skip_footer_rows) { should == 0 }
16
+ end
17
+
18
+ describe "initialize" do
19
+ it "configures options with argument hash" do
20
+ reader = Readers::CSV.new(col_sep: '|')
21
+ reader.col_sep.should == '|'
22
+ end
23
+
24
+ it "configures options with a block" do
25
+ reader = Readers::CSV.new do
26
+ encoding 'ISO-8859-1'
27
+ col_sep "|"
28
+ skip_header_rows 2
29
+ skip_footer_rows 1
30
+ end
31
+
32
+ reader.col_sep.should == '|'
33
+ reader.skip_header_rows.should == 2
34
+ reader.skip_footer_rows.should == 1
35
+ end
36
+ end
37
+
38
+ describe "#parse" do
39
+ it "returns an array of records" do
40
+ records = reader.parse "id,name\n42,Arthur Dent"
41
+
42
+ records[0].should == {"id" => "42", "name" => "Arthur Dent"}
43
+ end
44
+
45
+ it "strips null bytes from empty fields (MSSQL BCP exports)" do
46
+ reader.col_sep = '|'
47
+ reader.headers = %w[city state country]
48
+ records = reader.parse "Vancouver|\x00|Canada"
49
+
50
+ records.should ==
51
+ [{"city" => "Vancouver", "state" => nil, "country" => "Canada"}]
52
+ end
53
+
54
+ it "strips preamble header rows" do
55
+ reader.skip_header_rows = 2
56
+ data = "IGNORE\r\nCOMMENTS\r\nid,name\r\n42,Arthur Dent"
57
+ records = reader.parse data
58
+
59
+ records[0].should == {"id" => "42", "name" => "Arthur Dent"}
60
+ end
61
+
62
+ it "shares hash key instances between records" do
63
+ records = reader.parse "id,name\n42,Arthur Dent\n999,Zaphod"
64
+ records[0].keys[0].should be records[1].keys[0]
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,52 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/fixed_width'
3
+
4
+ module Stockboy
5
+ describe Readers::FixedWidth do
6
+
7
+ let(:headers_hash) { {first: 10, last: 10, age: 4, sex: 2, planet: 10} }
8
+ let(:blank_row) { " \r\n" }
9
+ let(:data) {
10
+ "Arthur Dent 42 M Human \r\n" +
11
+ "Ford Prefect 44 M Betelgeuse\r\n"
12
+ }
13
+
14
+ it "parses based on column widths" do
15
+ reader = described_class.new(headers: [10, 10, 4, 2, 10])
16
+ reader.parse(data).should == [
17
+ {0 => "Arthur", 1 => "Dent", 2 => "42", 3 => "M", 4 => "Human"},
18
+ {0 => "Ford", 1 => "Prefect", 2 => "44", 3 => "M", 4 => "Betelgeuse"}
19
+ ]
20
+ end
21
+
22
+ it "parses based on a hash" do
23
+ reader = described_class.new(headers: headers_hash)
24
+ reader.parse(data).should == [
25
+ {first: "Arthur", last: "Dent", age: "42", sex: "M", planet: "Human"},
26
+ {first: "Ford", last: "Prefect", age: "44", sex: "M", planet: "Betelgeuse"}
27
+ ]
28
+ end
29
+
30
+ it "skips blank rows" do
31
+ reader = described_class.new(headers: headers_hash)
32
+ records = reader.parse(blank_row + data + blank_row)
33
+ records.first[:age].should == '42'
34
+ records.last[:age].should == '44'
35
+ end
36
+
37
+ it "skips number of specified header rows" do
38
+ reader = described_class.new(headers: headers_hash)
39
+ reader.skip_header_rows = 1
40
+ records = reader.parse("REPORT\r\n" + data)
41
+ records.first[:age].should == '42'
42
+ end
43
+
44
+ it "skips number of specified footer rows" do
45
+ reader = described_class.new(headers: headers_hash)
46
+ reader.skip_footer_rows = 2
47
+ records = reader.parse(data + "SUBTOTAL\r\nTOTAL\r\n")
48
+ records.last[:age].should == '44'
49
+ end
50
+
51
+ end
52
+ end
@@ -0,0 +1,121 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/spreadsheet'
3
+
4
+ module Stockboy
5
+ describe Readers::Spreadsheet do
6
+
7
+ subject(:reader) { Stockboy::Readers::Spreadsheet.new }
8
+
9
+ describe "default options" do
10
+ its(:format) { should == :xls }
11
+ its(:sheet) { should == :first }
12
+ end
13
+
14
+
15
+ describe "#initialize" do
16
+ it "configures with hash parameter" do
17
+ reader = described_class.new(
18
+ format: :xlsx,
19
+ sheet: 'Sheet 42',
20
+ header_row: 5,
21
+ headers: %w(X Y Z)
22
+ )
23
+
24
+ reader.format.should == :xlsx
25
+ reader.sheet.should == 'Sheet 42'
26
+ reader.header_row.should == 5
27
+ reader.headers.should == %w(X Y Z)
28
+ end
29
+
30
+ it "configures with a block" do
31
+ reader = described_class.new do
32
+ format :xlsx
33
+ sheet 'Sheet 42'
34
+ header_row 5
35
+ headers %w(X Y Z)
36
+ end
37
+
38
+ reader.format.should == :xlsx
39
+ reader.sheet.should == 'Sheet 42'
40
+ reader.header_row.should == 5
41
+ reader.headers.should == %w(X Y Z)
42
+ end
43
+ end
44
+
45
+ describe "#parse" do
46
+ let(:content) { File.read(RSpec.configuration.fixture_path.join(fixture_file)) }
47
+
48
+ context "with an XLS file" do
49
+ let(:fixture_file) { 'spreadsheets/test_data.xls' }
50
+
51
+ it "returns an array of hashes" do
52
+ reader = described_class.new(format: :xls)
53
+ data = reader.parse(content)
54
+
55
+ data.should_not be_empty
56
+ data.each { |i| i.should be_a Hash }
57
+ end
58
+ end
59
+
60
+ context "with blank line options" do
61
+ let(:fixture_file) { 'spreadsheets/test_row_options.xls' }
62
+
63
+ it "starts on the given first row" do
64
+ reader = described_class.new(format: :xls, first_row: 6)
65
+ data = reader.parse(content)
66
+
67
+ data.first.values.should == ["Arthur Dent", 42]
68
+ end
69
+
70
+ it "ends on the given last row counting backwards" do
71
+ reader = described_class.new(format: :xls, last_row: -3)
72
+ data = reader.parse(content)
73
+
74
+ data.last.values.should == ["Marvin", 9999999]
75
+ end
76
+
77
+ it "ends on the given last row counting upwards" do
78
+ reader = described_class.new(format: :xls, last_row: 9)
79
+ data = reader.parse(content)
80
+
81
+ data.last.values.should == ["Ford", 40]
82
+ end
83
+ end
84
+ end
85
+
86
+ describe "#sheet", pending: "Hard to test this due to roo. Needs a test case with fixtures" do
87
+ let(:sheets) { ['Towels', 'Lemons'] }
88
+ let(:expectation) { mock(:spreadsheet, sheets: sheets).should_receive(:default_sheet=) }
89
+ let(:spreadsheet) { expectation }
90
+ before { subject.stub!(:with_spreadsheet).and_yield(spreadsheet) }
91
+
92
+ context "with :first" do
93
+ let(:spreadsheet) { expectation.with('Towels') }
94
+
95
+ it "calls on first sheet by name" do
96
+ subject.sheet = :first
97
+ subject.parse ""
98
+ end
99
+ end
100
+
101
+ context "with a string" do
102
+ let(:spreadsheet) { expectation.with('Towels') }
103
+
104
+ it "passes unchanged" do
105
+ subject.sheet = 'Towels'
106
+ subject.parse ""
107
+ end
108
+ end
109
+
110
+ context "with an integer" do
111
+ let(:spreadsheet) { expectation.with('Towels') }
112
+
113
+ it "gets sheet name" do
114
+ subject.sheet = 1
115
+ subject.parse ""
116
+ end
117
+ end
118
+ end
119
+
120
+ end
121
+ end
@@ -0,0 +1,94 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/xml'
3
+
4
+ module Stockboy
5
+ describe Readers::XML do
6
+ subject(:reader) { Stockboy::Readers::XML.new }
7
+
8
+ describe "initialize" do
9
+ it "initializes hash options" do
10
+ reader = Readers::XML.new(
11
+ strip_namespaces: true,
12
+ advanced_typecasting: true,
13
+ convert_tags_to: ->(t) { t.snakecase },
14
+ parser: :nokogiri,
15
+ elements: ['SomeNested', 'Record']
16
+ )
17
+
18
+ reader.options[:strip_namespaces].should be_true
19
+ reader.options[:advanced_typecasting].should be_true
20
+ reader.options[:convert_tags_to].should be_a Proc
21
+ reader.options[:parser].should == :nokogiri
22
+ reader.elements.should == ['some_nested', 'record']
23
+ end
24
+
25
+ it "configures with a block" do
26
+ reader = Readers::XML.new do
27
+ encoding 'UTF-8'
28
+ strip_namespaces true
29
+ advanced_typecasting true
30
+ convert_tags_to ->(t) { t.snakecase }
31
+ parser :nokogiri
32
+ elements ['SomeNested', 'Record']
33
+ end
34
+
35
+ reader.options[:strip_namespaces].should be_true
36
+ reader.options[:advanced_typecasting].should be_true
37
+ reader.options[:convert_tags_to].should be_a Proc
38
+ reader.options[:parser].should == :nokogiri
39
+ reader.elements.should == ['some_nested', 'record']
40
+ end
41
+ end
42
+
43
+ describe "#parse" do
44
+
45
+ let(:xml_file) { RSpec.configuration.fixture_path.join "xml/body.xml" }
46
+ let(:xml_string) { File.read(xml_file) }
47
+ let(:elements) { ['MultiNamespacedEntryResponse', 'history', 'case'] }
48
+ let(:output_keys) { ['logTime', 'logType', 'logText'] }
49
+ subject(:reader) { Readers::XML.new(elements: elements, strip_namespaces: true) }
50
+
51
+ shared_examples :parse_data do
52
+ it "returns element hashes" do
53
+ items = reader.parse data
54
+ output_keys.each do |k|
55
+ items[0].keys.should include k
56
+ end
57
+ end
58
+
59
+ it "shares key string instances between records" do
60
+ items = reader.parse data
61
+ output_keys.each do |key|
62
+ i = items[0].keys.index(key)
63
+ items[0].keys[i].should be items[1].keys[i]
64
+ end
65
+ end
66
+ end
67
+
68
+ shared_examples :parse_data_with_tag_conversion do
69
+ context "with tag conversion" do
70
+ before { reader.options[:convert_tags_to] = ->(t) { t.snakecase } }
71
+ let(:output_keys) { ['log_time', 'log_type', 'log_text'] }
72
+
73
+ include_examples :parse_data
74
+ end
75
+ end
76
+
77
+ context "an XML string" do
78
+ let(:data) { xml_string }
79
+
80
+ include_examples :parse_data
81
+ # include_examples :parse_data_with_tag_conversion
82
+ end
83
+
84
+
85
+ context "a SOAP response" do
86
+ let(:data) { Nori.new(strip_namespaces: true).parse(xml_string) }
87
+
88
+ include_examples :parse_data
89
+ include_examples :parse_data_with_tag_conversion
90
+ end
91
+
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers'
3
+
4
+ module Stockboy
5
+ describe Readers do
6
+
7
+ let(:reader_class) { Class.new }
8
+
9
+ describe ".register" do
10
+ it "registers a key for a reader class" do
11
+ Readers.register(:markup, reader_class).should == reader_class
12
+ end
13
+ end
14
+
15
+ describe ".find" do
16
+ it "returns a reader class" do
17
+ Readers.register(:markup, reader_class)
18
+ Readers.find(:markup).should == reader_class
19
+ end
20
+ end
21
+
22
+ describe ".[]" do
23
+ it "returns a reader class" do
24
+ Readers.register(:markup, reader_class)
25
+ Readers[:markup].should == reader_class
26
+ end
27
+ end
28
+
29
+ end
30
+ end
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/source_record'
3
+
4
+ module Stockboy
5
+ describe SourceRecord do
6
+ subject(:record) do
7
+ SourceRecord.new({:full_name => 'Arthur Dent'},
8
+ {3 => 'Arthur Dent'})
9
+ end
10
+
11
+ it "accesses initialized fields from hash" do
12
+ record.full_name.should == 'Arthur Dent'
13
+ end
14
+
15
+ it "accesses source field names" do
16
+ record[3].should == 'Arthur Dent'
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/template_file'
3
+
4
+ module Stockboy
5
+ describe TemplateFile do
6
+
7
+ let(:template_path) { File.expand_path("../../fixtures/jobs", __FILE__) }
8
+
9
+ before do
10
+ Stockboy.configuration.template_load_paths = [template_path]
11
+ end
12
+
13
+ describe ".read" do
14
+ it "returns the template string" do
15
+ TemplateFile.read("test_job").should match "# file exists!"
16
+ end
17
+ end
18
+
19
+ describe ".template_path" do
20
+ it "returns nil when missing" do
21
+ TemplateFile.find("not_here").should be_nil
22
+ end
23
+
24
+ it "returns a file path when found" do
25
+ TemplateFile.find("test_job").should == "#{template_path}/test_job.rb"
26
+ end
27
+ end
28
+
29
+ end
30
+ end