davidrichards-etl 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +261 -0
- data/VERSION.yml +4 -0
- data/bin/etl +27 -0
- data/lib/all.rb +4 -0
- data/lib/etl/active_record_loader.rb +50 -0
- data/lib/etl/bucket.rb +148 -0
- data/lib/etl/csv_et.rb +64 -0
- data/lib/etl/etl.rb +273 -0
- data/lib/etl/time_bucket.rb +104 -0
- data/lib/etl/xml_et.rb +6 -0
- data/lib/etl.rb +36 -0
- data/lib/helpers/array.rb +11 -0
- data/lib/helpers/observation.rb +10 -0
- data/lib/helpers/open_struct.rb +18 -0
- data/lib/helpers/string.rb +6 -0
- data/lib/helpers/symbol.rb +6 -0
- data/spec/etl/bucket_spec.rb +112 -0
- data/spec/etl/csv_et_spec.rb +43 -0
- data/spec/etl/etl_spec.rb +237 -0
- data/spec/etl/xml_et_spec.rb +50 -0
- data/spec/etl_spec.rb +16 -0
- data/spec/fixtures/test_file.csv +3 -0
- data/spec/helpers/array_spec.rb +13 -0
- data/spec/helpers/observation_spec.rb +22 -0
- data/spec/helpers/open_struct_spec.rb +25 -0
- data/spec/helpers/string_spec.rb +8 -0
- data/spec/helpers/symbol_spec.rb +7 -0
- data/spec/spec_helper.rb +15 -0
- metadata +106 -0
@@ -0,0 +1,50 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
require 'etl/xml_et'
|
3
|
+
|
4
|
+
describe XML::ET do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@xml_file = File.expand_path("#{File.dirname(__FILE__)}/../fixtures/test_file.xml")
|
8
|
+
end
|
9
|
+
|
10
|
+
# I don't know if I want to make this just like csv_et. What I'll do is finish the scorm stuff, then come back.
|
11
|
+
end
|
12
|
+
|
13
|
+
# before do
|
14
|
+
# @csv_file = File.expand_path("#{File.dirname(__FILE__)}/../fixtures/test_file.csv")
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# it "should be able to transform a csv file into an array of arrays" do
|
18
|
+
# @etl = CSV::ET.process(:source => @csv_file)
|
19
|
+
# @etl.data.should be_is_a(Array)
|
20
|
+
# @etl.data.size.should eql(3)
|
21
|
+
# @etl.data.first.should eql(["some", "data", "here"])
|
22
|
+
# @etl.data.last.should eql([4,5,6])
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# it "should be able to transforrm csv data into an array of arrays" do
|
26
|
+
# content = File.read(@csv_file)
|
27
|
+
# @etl = CSV::ET.process(:source => content)
|
28
|
+
# @etl.data.should be_is_a(Array)
|
29
|
+
# @etl.data.size.should eql(3)
|
30
|
+
# @etl.data.first.should eql(["some", "data", "here"])
|
31
|
+
# @etl.data.last.should eql([4,5,6])
|
32
|
+
# end
|
33
|
+
#
|
34
|
+
# it "should be able to pull the header out of the extracted data" do
|
35
|
+
# @etl = CSV::ET.process(:source => @csv_file, :extract_header => true)
|
36
|
+
# @etl.header.should eql(["some", "data", "here"])
|
37
|
+
# @etl.data.first.should eql([1,2,3])
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
# it "should be able to use the FasterCSV options" do
|
41
|
+
# FasterCSV::Converters[:foo] = lambda{|f| :foo }
|
42
|
+
# @etl = CSV::ET.process(
|
43
|
+
# :source => @csv_file,
|
44
|
+
# :extract_header => true,
|
45
|
+
# :parse_with => {:converters => :foo}
|
46
|
+
# )
|
47
|
+
# @etl.header.should eql([:foo, :foo, :foo])
|
48
|
+
# @etl.data.first.should eql([:foo, :foo, :foo])
|
49
|
+
# end
|
50
|
+
# end
|
data/spec/etl_spec.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe "Etl" do
|
4
|
+
it "should use rubygems" do
|
5
|
+
defined?(Gem).should eql('constant')
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should use ActiveSupport" do
|
9
|
+
defined?(ActiveSupport).should eql('constant')
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should use OpenStruct" do
|
13
|
+
defined?(OpenStruct).should eql('constant')
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe Array do
|
4
|
+
it "should be able to symbolize values" do
|
5
|
+
%w(this Is a teSt).symbolize_values.should eql([:this, :is, :a, :te_st])
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should be able to symbolize values in place" do
|
9
|
+
a = %w(this Is a teSt)
|
10
|
+
a.symbolize_values!
|
11
|
+
a.should eql([:this, :is, :a, :te_st])
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe Observation do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@o = Observation.new
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should be an OpenStruct" do
|
10
|
+
@o.should be_is_a(OpenStruct)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should set occurred_at" do
|
14
|
+
@o.occured_at.should be_close(Time.now, 0.0001)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should make a setter and getter for occured_at" do
|
18
|
+
t = Time.now - 100
|
19
|
+
@o.occured_at = t
|
20
|
+
@o.occured_at.should eql(t)
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe OpenStruct do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@o = OpenStruct.new(:this => :that)
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should make the table available." do
|
10
|
+
@o.table.should ==({:this => :that})
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should make the keys to the table available" do
|
14
|
+
@o.keys.should eql([:this])
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should make the values available" do
|
18
|
+
@o.values.should eql([:that])
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should be able to check if a key is included in the keys" do
|
22
|
+
@o.should be_include(:this)
|
23
|
+
@o.should_not be_include(:that)
|
24
|
+
end
|
25
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
$: << File.join(File.dirname(__FILE__), "/../lib")
|
2
|
+
require 'rubygems'
|
3
|
+
require 'spec'
|
4
|
+
require 'etl'
|
5
|
+
|
6
|
+
require 'tempfile'
|
7
|
+
ETL.logger_root = Dir.tmpdir
|
8
|
+
|
9
|
+
Spec::Runner.configure do |config|
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
def logger_contents
|
14
|
+
File.read(ETL.logger_filename)
|
15
|
+
end
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: davidrichards-etl
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Richards
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-08-03 00:00:00 -07:00
|
13
|
+
default_executable: etl
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: activesupport
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: log4r
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: "0"
|
34
|
+
version:
|
35
|
+
description: A basic ETL utility to make extract, transform, and load projects simpler, logged, and sharable
|
36
|
+
email: davidlamontrichards@gmail.com
|
37
|
+
executables:
|
38
|
+
- etl
|
39
|
+
extensions: []
|
40
|
+
|
41
|
+
extra_rdoc_files: []
|
42
|
+
|
43
|
+
files:
|
44
|
+
- README.rdoc
|
45
|
+
- VERSION.yml
|
46
|
+
- bin/etl
|
47
|
+
- lib/all.rb
|
48
|
+
- lib/etl
|
49
|
+
- lib/etl/active_record_loader.rb
|
50
|
+
- lib/etl/bucket.rb
|
51
|
+
- lib/etl/csv_et.rb
|
52
|
+
- lib/etl/etl.rb
|
53
|
+
- lib/etl/time_bucket.rb
|
54
|
+
- lib/etl/xml_et.rb
|
55
|
+
- lib/etl.rb
|
56
|
+
- lib/helpers
|
57
|
+
- lib/helpers/array.rb
|
58
|
+
- lib/helpers/observation.rb
|
59
|
+
- lib/helpers/open_struct.rb
|
60
|
+
- lib/helpers/string.rb
|
61
|
+
- lib/helpers/symbol.rb
|
62
|
+
- spec/etl
|
63
|
+
- spec/etl/bucket_spec.rb
|
64
|
+
- spec/etl/csv_et_spec.rb
|
65
|
+
- spec/etl/etl_spec.rb
|
66
|
+
- spec/etl/xml_et_spec.rb
|
67
|
+
- spec/etl_spec.rb
|
68
|
+
- spec/fixtures
|
69
|
+
- spec/fixtures/test_file.csv
|
70
|
+
- spec/helpers
|
71
|
+
- spec/helpers/array_spec.rb
|
72
|
+
- spec/helpers/observation_spec.rb
|
73
|
+
- spec/helpers/open_struct_spec.rb
|
74
|
+
- spec/helpers/string_spec.rb
|
75
|
+
- spec/helpers/symbol_spec.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
has_rdoc: true
|
78
|
+
homepage: http://github.com/davidrichards/etl
|
79
|
+
licenses:
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options:
|
82
|
+
- --inline-source
|
83
|
+
- --charset=UTF-8
|
84
|
+
require_paths:
|
85
|
+
- lib
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: "0"
|
91
|
+
version:
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: "0"
|
97
|
+
version:
|
98
|
+
requirements: []
|
99
|
+
|
100
|
+
rubyforge_project:
|
101
|
+
rubygems_version: 1.3.5
|
102
|
+
signing_key:
|
103
|
+
specification_version: 2
|
104
|
+
summary: A basic ETL utility to make extract, transform, and load projects simpler, logged, and sharable
|
105
|
+
test_files: []
|
106
|
+
|