petl 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/petl.rb +70 -1
- data/lib/petl/version.rb +1 -1
- data/spec/lib/{petl/etl_spec.rb → petl_spec.rb} +3 -3
- metadata +4 -5
- data/lib/petl/etl.rb +0 -71
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e81cfd5789c4763371eee08ef0b186011fdd7c94
|
4
|
+
data.tar.gz: b481436d739a7ec66cb07155edd2af5f0f2780fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9fe850cdf3e295e59c56030edaff1576c54f97b44f5dd333b68568fb99eb8c3e8a25511d73da905f98f7d4f9d71a89b91397f3fb9f135e5b6d135a201134afa
|
7
|
+
data.tar.gz: 03113c404ffa6a6a9226572db0974116ea96d14b0babd170ea2657b71ed699a8c25d23c558e43053f3624c257582aa1f2ea8a270897ee0abbb767a631ef38309
|
data/README.md
CHANGED
data/lib/petl.rb
CHANGED
@@ -1,2 +1,71 @@
|
|
1
1
|
require "petl/version"
|
2
|
-
|
2
|
+
|
3
|
+
module Petl
|
4
|
+
def perform logger = Rails.logger
|
5
|
+
if batch
|
6
|
+
batch_perform logger
|
7
|
+
else
|
8
|
+
not_batch_perform logger
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def extract
|
13
|
+
raise NotImplementedError.new "#{self}#extract not implemented."
|
14
|
+
end
|
15
|
+
|
16
|
+
def transform
|
17
|
+
raise NotImplementedError.new "#{self}#transform not implemented."
|
18
|
+
end
|
19
|
+
|
20
|
+
def load
|
21
|
+
raise NotImplementedError.new "#{self}#load not implemented."
|
22
|
+
end
|
23
|
+
|
24
|
+
def source_count
|
25
|
+
raise NotImplementedError.new "#{self}#source_count not implemented."
|
26
|
+
end
|
27
|
+
|
28
|
+
def destination_count
|
29
|
+
raise NotImplementedError.new "#{self}#destination_count not implemented."
|
30
|
+
end
|
31
|
+
|
32
|
+
def verify logger = Rails.logger
|
33
|
+
if source_count != destination_count
|
34
|
+
logger.error "#{self}: counts don't match"
|
35
|
+
end
|
36
|
+
|
37
|
+
logger.info "#{self}: source count #{source_count}"
|
38
|
+
logger.info "#{self}: destination count #{destination_count}"
|
39
|
+
end
|
40
|
+
|
41
|
+
def batch
|
42
|
+
false
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def not_batch_perform logger = Rails.logger
|
48
|
+
with_profiling logger do
|
49
|
+
load(transform(extract))
|
50
|
+
verify logger
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def batch_perform logger = Rails.logger
|
55
|
+
with_profiling logger do
|
56
|
+
extract do |batch|
|
57
|
+
load(transform batch)
|
58
|
+
end
|
59
|
+
verify logger
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def with_profiling logger, &block
|
64
|
+
start_time = Time.now
|
65
|
+
logger.info "#{self} starting at #{start_time}"
|
66
|
+
|
67
|
+
yield block
|
68
|
+
|
69
|
+
logger.info "#{self} finished at #{Time.now}. Took #{(Time.now - start_time).round} seconds"
|
70
|
+
end
|
71
|
+
end
|
data/lib/petl/version.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe "Petl
|
3
|
+
describe "Petl" do
|
4
4
|
|
5
5
|
module TestETL
|
6
|
-
extend Petl
|
6
|
+
extend Petl
|
7
7
|
extend self
|
8
8
|
|
9
9
|
def source_count
|
@@ -66,7 +66,7 @@ describe "Petl::ETL" do
|
|
66
66
|
|
67
67
|
context "with batching" do
|
68
68
|
module BatchETL
|
69
|
-
extend Petl
|
69
|
+
extend Petl
|
70
70
|
extend self
|
71
71
|
|
72
72
|
def batch
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: petl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Tamoykin
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-10-
|
12
|
+
date: 2013-10-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -69,10 +69,9 @@ files:
|
|
69
69
|
- README.md
|
70
70
|
- Rakefile
|
71
71
|
- lib/petl.rb
|
72
|
-
- lib/petl/etl.rb
|
73
72
|
- lib/petl/version.rb
|
74
73
|
- petl.gemspec
|
75
|
-
- spec/lib/
|
74
|
+
- spec/lib/petl_spec.rb
|
76
75
|
- spec/spec_helper.rb
|
77
76
|
homepage: ''
|
78
77
|
licenses:
|
@@ -99,5 +98,5 @@ signing_key:
|
|
99
98
|
specification_version: 4
|
100
99
|
summary: Pretty good ETL framework
|
101
100
|
test_files:
|
102
|
-
- spec/lib/
|
101
|
+
- spec/lib/petl_spec.rb
|
103
102
|
- spec/spec_helper.rb
|
data/lib/petl/etl.rb
DELETED
@@ -1,71 +0,0 @@
|
|
1
|
-
module Petl
|
2
|
-
module ETL
|
3
|
-
def perform logger = Rails.logger
|
4
|
-
if batch
|
5
|
-
batch_perform logger
|
6
|
-
else
|
7
|
-
not_batch_perform logger
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
def extract
|
12
|
-
raise NotImplementedError.new "#{self}#extract not implemented."
|
13
|
-
end
|
14
|
-
|
15
|
-
def transform
|
16
|
-
raise NotImplementedError.new "#{self}#transform not implemented."
|
17
|
-
end
|
18
|
-
|
19
|
-
def load
|
20
|
-
raise NotImplementedError.new "#{self}#load not implemented."
|
21
|
-
end
|
22
|
-
|
23
|
-
def source_count
|
24
|
-
raise NotImplementedError.new "#{self}#source_count not implemented."
|
25
|
-
end
|
26
|
-
|
27
|
-
def destination_count
|
28
|
-
raise NotImplementedError.new "#{self}#destination_count not implemented."
|
29
|
-
end
|
30
|
-
|
31
|
-
def verify logger = Rails.logger
|
32
|
-
if source_count != destination_count
|
33
|
-
logger.error "#{self}: counts don't match"
|
34
|
-
end
|
35
|
-
|
36
|
-
logger.info "#{self}: source count #{source_count}"
|
37
|
-
logger.info "#{self}: destination count #{destination_count}"
|
38
|
-
end
|
39
|
-
|
40
|
-
def batch
|
41
|
-
false
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
|
46
|
-
def not_batch_perform logger = Rails.logger
|
47
|
-
with_profiling logger do
|
48
|
-
load(transform(extract))
|
49
|
-
verify logger
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def batch_perform logger = Rails.logger
|
54
|
-
with_profiling logger do
|
55
|
-
extract do |batch|
|
56
|
-
load(transform batch)
|
57
|
-
end
|
58
|
-
verify logger
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
def with_profiling logger, &block
|
63
|
-
start_time = Time.now
|
64
|
-
logger.info "#{self} starting at #{start_time}"
|
65
|
-
|
66
|
-
yield block
|
67
|
-
|
68
|
-
logger.info "#{self} finished at #{Time.now}. Took #{(Time.now - start_time).round} seconds"
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|