davidrichards-etl 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +261 -0
- data/VERSION.yml +4 -0
- data/bin/etl +27 -0
- data/lib/all.rb +4 -0
- data/lib/etl/active_record_loader.rb +50 -0
- data/lib/etl/bucket.rb +148 -0
- data/lib/etl/csv_et.rb +64 -0
- data/lib/etl/etl.rb +273 -0
- data/lib/etl/time_bucket.rb +104 -0
- data/lib/etl/xml_et.rb +6 -0
- data/lib/etl.rb +36 -0
- data/lib/helpers/array.rb +11 -0
- data/lib/helpers/observation.rb +10 -0
- data/lib/helpers/open_struct.rb +18 -0
- data/lib/helpers/string.rb +6 -0
- data/lib/helpers/symbol.rb +6 -0
- data/spec/etl/bucket_spec.rb +112 -0
- data/spec/etl/csv_et_spec.rb +43 -0
- data/spec/etl/etl_spec.rb +237 -0
- data/spec/etl/xml_et_spec.rb +50 -0
- data/spec/etl_spec.rb +16 -0
- data/spec/fixtures/test_file.csv +3 -0
- data/spec/helpers/array_spec.rb +13 -0
- data/spec/helpers/observation_spec.rb +22 -0
- data/spec/helpers/open_struct_spec.rb +25 -0
- data/spec/helpers/string_spec.rb +8 -0
- data/spec/helpers/symbol_spec.rb +7 -0
- data/spec/spec_helper.rb +15 -0
- metadata +106 -0
@@ -0,0 +1,50 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
require 'etl/xml_et'
|
3
|
+
|
4
|
+
describe XML::ET do
|
5
|
+
|
6
|
+
before do
|
7
|
+
@xml_file = File.expand_path("#{File.dirname(__FILE__)}/../fixtures/test_file.xml")
|
8
|
+
end
|
9
|
+
|
10
|
+
# I don't know if I want to make this just like csv_et. What I'll do is finish the scorm stuff, then come back.
|
11
|
+
end
|
12
|
+
|
13
|
+
# before do
|
14
|
+
# @csv_file = File.expand_path("#{File.dirname(__FILE__)}/../fixtures/test_file.csv")
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# it "should be able to transform a csv file into an array of arrays" do
|
18
|
+
# @etl = CSV::ET.process(:source => @csv_file)
|
19
|
+
# @etl.data.should be_is_a(Array)
|
20
|
+
# @etl.data.size.should eql(3)
|
21
|
+
# @etl.data.first.should eql(["some", "data", "here"])
|
22
|
+
# @etl.data.last.should eql([4,5,6])
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# it "should be able to transforrm csv data into an array of arrays" do
|
26
|
+
# content = File.read(@csv_file)
|
27
|
+
# @etl = CSV::ET.process(:source => content)
|
28
|
+
# @etl.data.should be_is_a(Array)
|
29
|
+
# @etl.data.size.should eql(3)
|
30
|
+
# @etl.data.first.should eql(["some", "data", "here"])
|
31
|
+
# @etl.data.last.should eql([4,5,6])
|
32
|
+
# end
|
33
|
+
#
|
34
|
+
# it "should be able to pull the header out of the extracted data" do
|
35
|
+
# @etl = CSV::ET.process(:source => @csv_file, :extract_header => true)
|
36
|
+
# @etl.header.should eql(["some", "data", "here"])
|
37
|
+
# @etl.data.first.should eql([1,2,3])
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
# it "should be able to use the FasterCSV options" do
|
41
|
+
# FasterCSV::Converters[:foo] = lambda{|f| :foo }
|
42
|
+
# @etl = CSV::ET.process(
|
43
|
+
# :source => @csv_file,
|
44
|
+
# :extract_header => true,
|
45
|
+
# :parse_with => {:converters => :foo}
|
46
|
+
# )
|
47
|
+
# @etl.header.should eql([:foo, :foo, :foo])
|
48
|
+
# @etl.data.first.should eql([:foo, :foo, :foo])
|
49
|
+
# end
|
50
|
+
# end
|
data/spec/etl_spec.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe "Etl" do
|
4
|
+
it "should use rubygems" do
|
5
|
+
defined?(Gem).should eql('constant')
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should use ActiveSupport" do
|
9
|
+
defined?(ActiveSupport).should eql('constant')
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should use OpenStruct" do
|
13
|
+
defined?(OpenStruct).should eql('constant')
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe Array do
|
4
|
+
it "should be able to symbolize values" do
|
5
|
+
%w(this Is a teSt).symbolize_values.should eql([:this, :is, :a, :te_st])
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should be able to symbolize values in place" do
|
9
|
+
a = %w(this Is a teSt)
|
10
|
+
a.symbolize_values!
|
11
|
+
a.should eql([:this, :is, :a, :te_st])
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe Observation do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@o = Observation.new
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should be an OpenStruct" do
|
10
|
+
@o.should be_is_a(OpenStruct)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should set occurred_at" do
|
14
|
+
@o.occured_at.should be_close(Time.now, 0.0001)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should make a setter and getter for occured_at" do
|
18
|
+
t = Time.now - 100
|
19
|
+
@o.occured_at = t
|
20
|
+
@o.occured_at.should eql(t)
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "/../spec_helper")
|
2
|
+
|
3
|
+
describe OpenStruct do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@o = OpenStruct.new(:this => :that)
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should make the table available." do
|
10
|
+
@o.table.should ==({:this => :that})
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should make the keys to the table available" do
|
14
|
+
@o.keys.should eql([:this])
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should make the values available" do
|
18
|
+
@o.values.should eql([:that])
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should be able to check if a key is included in the keys" do
|
22
|
+
@o.should be_include(:this)
|
23
|
+
@o.should_not be_include(:that)
|
24
|
+
end
|
25
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
$: << File.join(File.dirname(__FILE__), "/../lib")
|
2
|
+
require 'rubygems'
|
3
|
+
require 'spec'
|
4
|
+
require 'etl'
|
5
|
+
|
6
|
+
require 'tempfile'
|
7
|
+
ETL.logger_root = Dir.tmpdir
|
8
|
+
|
9
|
+
Spec::Runner.configure do |config|
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
def logger_contents
|
14
|
+
File.read(ETL.logger_filename)
|
15
|
+
end
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: davidrichards-etl
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Richards
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-08-03 00:00:00 -07:00
|
13
|
+
default_executable: etl
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: activesupport
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: log4r
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: "0"
|
34
|
+
version:
|
35
|
+
description: A basic ETL utility to make extract, transform, and load projects simpler, logged, and sharable
|
36
|
+
email: davidlamontrichards@gmail.com
|
37
|
+
executables:
|
38
|
+
- etl
|
39
|
+
extensions: []
|
40
|
+
|
41
|
+
extra_rdoc_files: []
|
42
|
+
|
43
|
+
files:
|
44
|
+
- README.rdoc
|
45
|
+
- VERSION.yml
|
46
|
+
- bin/etl
|
47
|
+
- lib/all.rb
|
48
|
+
- lib/etl
|
49
|
+
- lib/etl/active_record_loader.rb
|
50
|
+
- lib/etl/bucket.rb
|
51
|
+
- lib/etl/csv_et.rb
|
52
|
+
- lib/etl/etl.rb
|
53
|
+
- lib/etl/time_bucket.rb
|
54
|
+
- lib/etl/xml_et.rb
|
55
|
+
- lib/etl.rb
|
56
|
+
- lib/helpers
|
57
|
+
- lib/helpers/array.rb
|
58
|
+
- lib/helpers/observation.rb
|
59
|
+
- lib/helpers/open_struct.rb
|
60
|
+
- lib/helpers/string.rb
|
61
|
+
- lib/helpers/symbol.rb
|
62
|
+
- spec/etl
|
63
|
+
- spec/etl/bucket_spec.rb
|
64
|
+
- spec/etl/csv_et_spec.rb
|
65
|
+
- spec/etl/etl_spec.rb
|
66
|
+
- spec/etl/xml_et_spec.rb
|
67
|
+
- spec/etl_spec.rb
|
68
|
+
- spec/fixtures
|
69
|
+
- spec/fixtures/test_file.csv
|
70
|
+
- spec/helpers
|
71
|
+
- spec/helpers/array_spec.rb
|
72
|
+
- spec/helpers/observation_spec.rb
|
73
|
+
- spec/helpers/open_struct_spec.rb
|
74
|
+
- spec/helpers/string_spec.rb
|
75
|
+
- spec/helpers/symbol_spec.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
has_rdoc: true
|
78
|
+
homepage: http://github.com/davidrichards/etl
|
79
|
+
licenses:
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options:
|
82
|
+
- --inline-source
|
83
|
+
- --charset=UTF-8
|
84
|
+
require_paths:
|
85
|
+
- lib
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: "0"
|
91
|
+
version:
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: "0"
|
97
|
+
version:
|
98
|
+
requirements: []
|
99
|
+
|
100
|
+
rubyforge_project:
|
101
|
+
rubygems_version: 1.3.5
|
102
|
+
signing_key:
|
103
|
+
specification_version: 2
|
104
|
+
summary: A basic ETL utility to make extract, transform, and load projects simpler, logged, and sharable
|
105
|
+
test_files: []
|
106
|
+
|