stockboy 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.rspec +5 -0
  4. data/.yardopts +7 -0
  5. data/CHANGELOG.md +24 -0
  6. data/Gemfile +12 -0
  7. data/Guardfile +10 -0
  8. data/LICENSE +21 -0
  9. data/README.md +293 -0
  10. data/Rakefile +30 -0
  11. data/lib/stockboy.rb +80 -0
  12. data/lib/stockboy/attribute.rb +11 -0
  13. data/lib/stockboy/attribute_map.rb +74 -0
  14. data/lib/stockboy/candidate_record.rb +130 -0
  15. data/lib/stockboy/configuration.rb +62 -0
  16. data/lib/stockboy/configurator.rb +176 -0
  17. data/lib/stockboy/dsl.rb +68 -0
  18. data/lib/stockboy/exceptions.rb +3 -0
  19. data/lib/stockboy/filter.rb +58 -0
  20. data/lib/stockboy/filter_chain.rb +41 -0
  21. data/lib/stockboy/filters.rb +11 -0
  22. data/lib/stockboy/filters/missing_email.rb +37 -0
  23. data/lib/stockboy/job.rb +241 -0
  24. data/lib/stockboy/mapped_record.rb +59 -0
  25. data/lib/stockboy/provider.rb +238 -0
  26. data/lib/stockboy/providers.rb +11 -0
  27. data/lib/stockboy/providers/file.rb +135 -0
  28. data/lib/stockboy/providers/ftp.rb +205 -0
  29. data/lib/stockboy/providers/http.rb +123 -0
  30. data/lib/stockboy/providers/imap.rb +290 -0
  31. data/lib/stockboy/providers/soap.rb +120 -0
  32. data/lib/stockboy/railtie.rb +28 -0
  33. data/lib/stockboy/reader.rb +59 -0
  34. data/lib/stockboy/readers.rb +11 -0
  35. data/lib/stockboy/readers/csv.rb +115 -0
  36. data/lib/stockboy/readers/fixed_width.rb +121 -0
  37. data/lib/stockboy/readers/spreadsheet.rb +144 -0
  38. data/lib/stockboy/readers/xml.rb +155 -0
  39. data/lib/stockboy/registry.rb +42 -0
  40. data/lib/stockboy/source_record.rb +43 -0
  41. data/lib/stockboy/string_pool.rb +35 -0
  42. data/lib/stockboy/template_file.rb +44 -0
  43. data/lib/stockboy/translations.rb +70 -0
  44. data/lib/stockboy/translations/boolean.rb +58 -0
  45. data/lib/stockboy/translations/date.rb +41 -0
  46. data/lib/stockboy/translations/decimal.rb +33 -0
  47. data/lib/stockboy/translations/default_empty_string.rb +38 -0
  48. data/lib/stockboy/translations/default_false.rb +41 -0
  49. data/lib/stockboy/translations/default_nil.rb +38 -0
  50. data/lib/stockboy/translations/default_true.rb +41 -0
  51. data/lib/stockboy/translations/default_zero.rb +41 -0
  52. data/lib/stockboy/translations/integer.rb +33 -0
  53. data/lib/stockboy/translations/string.rb +33 -0
  54. data/lib/stockboy/translations/time.rb +41 -0
  55. data/lib/stockboy/translations/uk_date.rb +51 -0
  56. data/lib/stockboy/translations/us_date.rb +51 -0
  57. data/lib/stockboy/translator.rb +66 -0
  58. data/lib/stockboy/version.rb +3 -0
  59. data/spec/fixtures/.gitkeep +0 -0
  60. data/spec/fixtures/files/a_garbage.csv +1 -0
  61. data/spec/fixtures/files/test_data-20120101.csv +1 -0
  62. data/spec/fixtures/files/test_data-20120202.csv +1 -0
  63. data/spec/fixtures/files/z_garbage.csv +1 -0
  64. data/spec/fixtures/jobs/test_job.rb +1 -0
  65. data/spec/fixtures/soap/get_list/fault.xml +8 -0
  66. data/spec/fixtures/soap/get_list/success.xml +18 -0
  67. data/spec/fixtures/spreadsheets/test_data.xls +0 -0
  68. data/spec/fixtures/spreadsheets/test_row_options.xls +0 -0
  69. data/spec/fixtures/xml/body.xml +14 -0
  70. data/spec/spec_helper.rb +28 -0
  71. data/spec/stockboy/attribute_map_spec.rb +59 -0
  72. data/spec/stockboy/attribute_spec.rb +11 -0
  73. data/spec/stockboy/candidate_record_spec.rb +150 -0
  74. data/spec/stockboy/configuration_spec.rb +28 -0
  75. data/spec/stockboy/configurator_spec.rb +127 -0
  76. data/spec/stockboy/filter_chain_spec.rb +40 -0
  77. data/spec/stockboy/filter_spec.rb +41 -0
  78. data/spec/stockboy/filters/missing_email_spec.rb +26 -0
  79. data/spec/stockboy/filters_spec.rb +38 -0
  80. data/spec/stockboy/job_spec.rb +238 -0
  81. data/spec/stockboy/mapped_record_spec.rb +30 -0
  82. data/spec/stockboy/provider_spec.rb +34 -0
  83. data/spec/stockboy/providers/file_spec.rb +116 -0
  84. data/spec/stockboy/providers/ftp_spec.rb +143 -0
  85. data/spec/stockboy/providers/http_spec.rb +94 -0
  86. data/spec/stockboy/providers/imap_spec.rb +76 -0
  87. data/spec/stockboy/providers/soap_spec.rb +107 -0
  88. data/spec/stockboy/providers_spec.rb +38 -0
  89. data/spec/stockboy/readers/csv_spec.rb +68 -0
  90. data/spec/stockboy/readers/fixed_width_spec.rb +52 -0
  91. data/spec/stockboy/readers/spreadsheet_spec.rb +121 -0
  92. data/spec/stockboy/readers/xml_spec.rb +94 -0
  93. data/spec/stockboy/readers_spec.rb +30 -0
  94. data/spec/stockboy/source_record_spec.rb +19 -0
  95. data/spec/stockboy/template_file_spec.rb +30 -0
  96. data/spec/stockboy/translations/boolean_spec.rb +48 -0
  97. data/spec/stockboy/translations/date_spec.rb +38 -0
  98. data/spec/stockboy/translations/decimal_spec.rb +23 -0
  99. data/spec/stockboy/translations/default_empty_string_spec.rb +32 -0
  100. data/spec/stockboy/translations/default_false_spec.rb +25 -0
  101. data/spec/stockboy/translations/default_nil_spec.rb +32 -0
  102. data/spec/stockboy/translations/default_true_spec.rb +25 -0
  103. data/spec/stockboy/translations/default_zero_spec.rb +32 -0
  104. data/spec/stockboy/translations/integer_spec.rb +22 -0
  105. data/spec/stockboy/translations/string_spec.rb +22 -0
  106. data/spec/stockboy/translations/time_spec.rb +27 -0
  107. data/spec/stockboy/translations/uk_date_spec.rb +37 -0
  108. data/spec/stockboy/translations/us_date_spec.rb +37 -0
  109. data/spec/stockboy/translations_spec.rb +55 -0
  110. data/spec/stockboy/translator_spec.rb +27 -0
  111. data/stockboy.gemspec +32 -0
  112. metadata +305 -0
@@ -0,0 +1,38 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/providers'
3
+
4
+ module Stockboy
5
+ describe Providers do
6
+
7
+ let(:provider) { double("provider") }
8
+
9
+ describe ".register" do
10
+ it "registers a key and class" do
11
+ Providers.register(:snailmail, provider).should === provider
12
+ end
13
+ end
14
+
15
+ describe ".find" do
16
+ it "returns a provider class" do
17
+ Providers.register(:snailmail, provider)
18
+ Providers.find(:snailmail).should === provider
19
+ end
20
+ end
21
+
22
+ describe ".[]" do
23
+ it "returns a provider class" do
24
+ Providers.register(:snailmail, provider)
25
+ Providers[:snailmail].should === provider
26
+ end
27
+ end
28
+
29
+ describe ".all" do
30
+ it "returns all registered providers" do
31
+ Providers.register(:snailmail, provider)
32
+ Providers.register(:slugmail, provider)
33
+ Providers.all.should include(snailmail: provider, slugmail: provider)
34
+ end
35
+ end
36
+
37
+ end
38
+ end
@@ -0,0 +1,68 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/csv'
3
+
4
+ module Stockboy
5
+ describe Readers::CSV do
6
+
7
+ subject(:reader) { Stockboy::Readers::CSV.new }
8
+
9
+ describe "default options" do
10
+ its(:row_sep) { should be_nil }
11
+ its(:col_sep) { should be_nil }
12
+ its(:quote_char) { should be_nil }
13
+ its(:headers) { should be_true }
14
+ its(:skip_header_rows) { should == 0 }
15
+ its(:skip_footer_rows) { should == 0 }
16
+ end
17
+
18
+ describe "initialize" do
19
+ it "configures options with argument hash" do
20
+ reader = Readers::CSV.new(col_sep: '|')
21
+ reader.col_sep.should == '|'
22
+ end
23
+
24
+ it "configures options with a block" do
25
+ reader = Readers::CSV.new do
26
+ encoding 'ISO-8859-1'
27
+ col_sep "|"
28
+ skip_header_rows 2
29
+ skip_footer_rows 1
30
+ end
31
+
32
+ reader.col_sep.should == '|'
33
+ reader.skip_header_rows.should == 2
34
+ reader.skip_footer_rows.should == 1
35
+ end
36
+ end
37
+
38
+ describe "#parse" do
39
+ it "returns an array of records" do
40
+ records = reader.parse "id,name\n42,Arthur Dent"
41
+
42
+ records[0].should == {"id" => "42", "name" => "Arthur Dent"}
43
+ end
44
+
45
+ it "strips null bytes from empty fields (MSSQL BCP exports)" do
46
+ reader.col_sep = '|'
47
+ reader.headers = %w[city state country]
48
+ records = reader.parse "Vancouver|\x00|Canada"
49
+
50
+ records.should ==
51
+ [{"city" => "Vancouver", "state" => nil, "country" => "Canada"}]
52
+ end
53
+
54
+ it "strips preamble header rows" do
55
+ reader.skip_header_rows = 2
56
+ data = "IGNORE\r\nCOMMENTS\r\nid,name\r\n42,Arthur Dent"
57
+ records = reader.parse data
58
+
59
+ records[0].should == {"id" => "42", "name" => "Arthur Dent"}
60
+ end
61
+
62
+ it "shares hash key instances between records" do
63
+ records = reader.parse "id,name\n42,Arthur Dent\n999,Zaphod"
64
+ records[0].keys[0].should be records[1].keys[0]
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,52 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/fixed_width'
3
+
4
+ module Stockboy
5
+ describe Readers::FixedWidth do
6
+
7
+ let(:headers_hash) { {first: 10, last: 10, age: 4, sex: 2, planet: 10} }
8
+ let(:blank_row) { " \r\n" }
9
+ let(:data) {
10
+ "Arthur Dent 42 M Human \r\n" +
11
+ "Ford Prefect 44 M Betelgeuse\r\n"
12
+ }
13
+
14
+ it "parses based on column widths" do
15
+ reader = described_class.new(headers: [10, 10, 4, 2, 10])
16
+ reader.parse(data).should == [
17
+ {0 => "Arthur", 1 => "Dent", 2 => "42", 3 => "M", 4 => "Human"},
18
+ {0 => "Ford", 1 => "Prefect", 2 => "44", 3 => "M", 4 => "Betelgeuse"}
19
+ ]
20
+ end
21
+
22
+ it "parses based on a hash" do
23
+ reader = described_class.new(headers: headers_hash)
24
+ reader.parse(data).should == [
25
+ {first: "Arthur", last: "Dent", age: "42", sex: "M", planet: "Human"},
26
+ {first: "Ford", last: "Prefect", age: "44", sex: "M", planet: "Betelgeuse"}
27
+ ]
28
+ end
29
+
30
+ it "skips blank rows" do
31
+ reader = described_class.new(headers: headers_hash)
32
+ records = reader.parse(blank_row + data + blank_row)
33
+ records.first[:age].should == '42'
34
+ records.last[:age].should == '44'
35
+ end
36
+
37
+ it "skips number of specified header rows" do
38
+ reader = described_class.new(headers: headers_hash)
39
+ reader.skip_header_rows = 1
40
+ records = reader.parse("REPORT\r\n" + data)
41
+ records.first[:age].should == '42'
42
+ end
43
+
44
+ it "skips number of specified footer rows" do
45
+ reader = described_class.new(headers: headers_hash)
46
+ reader.skip_footer_rows = 2
47
+ records = reader.parse(data + "SUBTOTAL\r\nTOTAL\r\n")
48
+ records.last[:age].should == '44'
49
+ end
50
+
51
+ end
52
+ end
@@ -0,0 +1,121 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/spreadsheet'
3
+
4
+ module Stockboy
5
+ describe Readers::Spreadsheet do
6
+
7
+ subject(:reader) { Stockboy::Readers::Spreadsheet.new }
8
+
9
+ describe "default options" do
10
+ its(:format) { should == :xls }
11
+ its(:sheet) { should == :first }
12
+ end
13
+
14
+
15
+ describe "#initialize" do
16
+ it "configures with hash parameter" do
17
+ reader = described_class.new(
18
+ format: :xlsx,
19
+ sheet: 'Sheet 42',
20
+ header_row: 5,
21
+ headers: %w(X Y Z)
22
+ )
23
+
24
+ reader.format.should == :xlsx
25
+ reader.sheet.should == 'Sheet 42'
26
+ reader.header_row.should == 5
27
+ reader.headers.should == %w(X Y Z)
28
+ end
29
+
30
+ it "configures with a block" do
31
+ reader = described_class.new do
32
+ format :xlsx
33
+ sheet 'Sheet 42'
34
+ header_row 5
35
+ headers %w(X Y Z)
36
+ end
37
+
38
+ reader.format.should == :xlsx
39
+ reader.sheet.should == 'Sheet 42'
40
+ reader.header_row.should == 5
41
+ reader.headers.should == %w(X Y Z)
42
+ end
43
+ end
44
+
45
+ describe "#parse" do
46
+ let(:content) { File.read(RSpec.configuration.fixture_path.join(fixture_file)) }
47
+
48
+ context "with an XLS file" do
49
+ let(:fixture_file) { 'spreadsheets/test_data.xls' }
50
+
51
+ it "returns an array of hashes" do
52
+ reader = described_class.new(format: :xls)
53
+ data = reader.parse(content)
54
+
55
+ data.should_not be_empty
56
+ data.each { |i| i.should be_a Hash }
57
+ end
58
+ end
59
+
60
+ context "with blank line options" do
61
+ let(:fixture_file) { 'spreadsheets/test_row_options.xls' }
62
+
63
+ it "starts on the given first row" do
64
+ reader = described_class.new(format: :xls, first_row: 6)
65
+ data = reader.parse(content)
66
+
67
+ data.first.values.should == ["Arthur Dent", 42]
68
+ end
69
+
70
+ it "ends on the given last row counting backwards" do
71
+ reader = described_class.new(format: :xls, last_row: -3)
72
+ data = reader.parse(content)
73
+
74
+ data.last.values.should == ["Marvin", 9999999]
75
+ end
76
+
77
+ it "ends on the given last row counting upwards" do
78
+ reader = described_class.new(format: :xls, last_row: 9)
79
+ data = reader.parse(content)
80
+
81
+ data.last.values.should == ["Ford", 40]
82
+ end
83
+ end
84
+ end
85
+
86
+ describe "#sheet", pending: "Hard to test this due to roo. Needs a test case with fixtures" do
87
+ let(:sheets) { ['Towels', 'Lemons'] }
88
+ let(:expectation) { mock(:spreadsheet, sheets: sheets).should_receive(:default_sheet=) }
89
+ let(:spreadsheet) { expectation }
90
+ before { subject.stub!(:with_spreadsheet).and_yield(spreadsheet) }
91
+
92
+ context "with :first" do
93
+ let(:spreadsheet) { expectation.with('Towels') }
94
+
95
+ it "calls on first sheet by name" do
96
+ subject.sheet = :first
97
+ subject.parse ""
98
+ end
99
+ end
100
+
101
+ context "with a string" do
102
+ let(:spreadsheet) { expectation.with('Towels') }
103
+
104
+ it "passes unchanged" do
105
+ subject.sheet = 'Towels'
106
+ subject.parse ""
107
+ end
108
+ end
109
+
110
+ context "with an integer" do
111
+ let(:spreadsheet) { expectation.with('Towels') }
112
+
113
+ it "gets sheet name" do
114
+ subject.sheet = 1
115
+ subject.parse ""
116
+ end
117
+ end
118
+ end
119
+
120
+ end
121
+ end
@@ -0,0 +1,94 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers/xml'
3
+
4
+ module Stockboy
5
+ describe Readers::XML do
6
+ subject(:reader) { Stockboy::Readers::XML.new }
7
+
8
+ describe "initialize" do
9
+ it "initializes hash options" do
10
+ reader = Readers::XML.new(
11
+ strip_namespaces: true,
12
+ advanced_typecasting: true,
13
+ convert_tags_to: ->(t) { t.snakecase },
14
+ parser: :nokogiri,
15
+ elements: ['SomeNested', 'Record']
16
+ )
17
+
18
+ reader.options[:strip_namespaces].should be_true
19
+ reader.options[:advanced_typecasting].should be_true
20
+ reader.options[:convert_tags_to].should be_a Proc
21
+ reader.options[:parser].should == :nokogiri
22
+ reader.elements.should == ['some_nested', 'record']
23
+ end
24
+
25
+ it "configures with a block" do
26
+ reader = Readers::XML.new do
27
+ encoding 'UTF-8'
28
+ strip_namespaces true
29
+ advanced_typecasting true
30
+ convert_tags_to ->(t) { t.snakecase }
31
+ parser :nokogiri
32
+ elements ['SomeNested', 'Record']
33
+ end
34
+
35
+ reader.options[:strip_namespaces].should be_true
36
+ reader.options[:advanced_typecasting].should be_true
37
+ reader.options[:convert_tags_to].should be_a Proc
38
+ reader.options[:parser].should == :nokogiri
39
+ reader.elements.should == ['some_nested', 'record']
40
+ end
41
+ end
42
+
43
+ describe "#parse" do
44
+
45
+ let(:xml_file) { RSpec.configuration.fixture_path.join "xml/body.xml" }
46
+ let(:xml_string) { File.read(xml_file) }
47
+ let(:elements) { ['MultiNamespacedEntryResponse', 'history', 'case'] }
48
+ let(:output_keys) { ['logTime', 'logType', 'logText'] }
49
+ subject(:reader) { Readers::XML.new(elements: elements, strip_namespaces: true) }
50
+
51
+ shared_examples :parse_data do
52
+ it "returns element hashes" do
53
+ items = reader.parse data
54
+ output_keys.each do |k|
55
+ items[0].keys.should include k
56
+ end
57
+ end
58
+
59
+ it "shares key string instances between records" do
60
+ items = reader.parse data
61
+ output_keys.each do |key|
62
+ i = items[0].keys.index(key)
63
+ items[0].keys[i].should be items[1].keys[i]
64
+ end
65
+ end
66
+ end
67
+
68
+ shared_examples :parse_data_with_tag_conversion do
69
+ context "with tag conversion" do
70
+ before { reader.options[:convert_tags_to] = ->(t) { t.snakecase } }
71
+ let(:output_keys) { ['log_time', 'log_type', 'log_text'] }
72
+
73
+ include_examples :parse_data
74
+ end
75
+ end
76
+
77
+ context "an XML string" do
78
+ let(:data) { xml_string }
79
+
80
+ include_examples :parse_data
81
+ # include_examples :parse_data_with_tag_conversion
82
+ end
83
+
84
+
85
+ context "a SOAP response" do
86
+ let(:data) { Nori.new(strip_namespaces: true).parse(xml_string) }
87
+
88
+ include_examples :parse_data
89
+ include_examples :parse_data_with_tag_conversion
90
+ end
91
+
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/readers'
3
+
4
+ module Stockboy
5
+ describe Readers do
6
+
7
+ let(:reader_class) { Class.new }
8
+
9
+ describe ".register" do
10
+ it "registers a key for a reader class" do
11
+ Readers.register(:markup, reader_class).should == reader_class
12
+ end
13
+ end
14
+
15
+ describe ".find" do
16
+ it "returns a reader class" do
17
+ Readers.register(:markup, reader_class)
18
+ Readers.find(:markup).should == reader_class
19
+ end
20
+ end
21
+
22
+ describe ".[]" do
23
+ it "returns a reader class" do
24
+ Readers.register(:markup, reader_class)
25
+ Readers[:markup].should == reader_class
26
+ end
27
+ end
28
+
29
+ end
30
+ end
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/source_record'
3
+
4
+ module Stockboy
5
+ describe SourceRecord do
6
+ subject(:record) do
7
+ SourceRecord.new({:full_name => 'Arthur Dent'},
8
+ {3 => 'Arthur Dent'})
9
+ end
10
+
11
+ it "accesses initialized fields from hash" do
12
+ record.full_name.should == 'Arthur Dent'
13
+ end
14
+
15
+ it "accesses source field names" do
16
+ record[3].should == 'Arthur Dent'
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'stockboy/template_file'
3
+
4
+ module Stockboy
5
+ describe TemplateFile do
6
+
7
+ let(:template_path) { File.expand_path("../../fixtures/jobs", __FILE__) }
8
+
9
+ before do
10
+ Stockboy.configuration.template_load_paths = [template_path]
11
+ end
12
+
13
+ describe ".read" do
14
+ it "returns the template string" do
15
+ TemplateFile.read("test_job").should match "# file exists!"
16
+ end
17
+ end
18
+
19
+ describe ".template_path" do
20
+ it "returns nil when missing" do
21
+ TemplateFile.find("not_here").should be_nil
22
+ end
23
+
24
+ it "returns a file path when found" do
25
+ TemplateFile.find("test_job").should == "#{template_path}/test_job.rb"
26
+ end
27
+ end
28
+
29
+ end
30
+ end