beso 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/beso.gemspec +2 -1
- data/gemfiles/rails-3.0.10.gemfile.lock +21 -2
- data/lib/beso/config.rb +23 -10
- data/lib/beso/connection.rb +65 -0
- data/lib/beso/csv.rb +22 -0
- data/lib/beso/job.rb +69 -26
- data/lib/beso/railtie.rb +1 -1
- data/lib/beso/version.rb +1 -1
- data/lib/beso.rb +15 -3
- data/lib/tasks/beso.rake +48 -0
- data/spec/beso/aws_spec.rb +16 -0
- data/spec/beso/config_spec.rb +60 -23
- data/spec/beso/job_spec.rb +230 -28
- data/spec/spec_helper.rb +4 -0
- metadata +26 -6
- data/lib/beso.rake +0 -6
data/beso.gemspec
CHANGED
@@ -16,7 +16,8 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.version = Beso::VERSION
|
17
17
|
|
18
18
|
gem.add_dependency 'rails', '>= 3.0.10'
|
19
|
-
gem.add_dependency '
|
19
|
+
gem.add_dependency 'fastercsv', '>= 1.5.4'
|
20
|
+
gem.add_dependency 'fog', '>= 1.3.1'
|
20
21
|
|
21
22
|
gem.add_development_dependency 'sqlite3'
|
22
23
|
gem.add_development_dependency 'appraisal', '>= 0.4.1'
|
@@ -2,7 +2,8 @@ PATH
|
|
2
2
|
remote: /Users/jeremyruppel/Git/beso
|
3
3
|
specs:
|
4
4
|
beso (0.0.1)
|
5
|
-
|
5
|
+
fastercsv (>= 1.5.4)
|
6
|
+
fog (>= 1.3.1)
|
6
7
|
rails (>= 3.0.10)
|
7
8
|
|
8
9
|
GEM
|
@@ -40,10 +41,22 @@ GEM
|
|
40
41
|
rake
|
41
42
|
arel (2.0.10)
|
42
43
|
builder (2.1.2)
|
43
|
-
comma (3.0.3)
|
44
44
|
diff-lcs (1.1.3)
|
45
45
|
erubis (2.6.6)
|
46
46
|
abstract (>= 1.0.0)
|
47
|
+
excon (0.13.3)
|
48
|
+
fastercsv (1.5.4)
|
49
|
+
fog (1.3.1)
|
50
|
+
builder
|
51
|
+
excon (~> 0.13.0)
|
52
|
+
formatador (~> 0.2.0)
|
53
|
+
mime-types
|
54
|
+
multi_json (~> 1.0)
|
55
|
+
net-scp (~> 1.0.4)
|
56
|
+
net-ssh (>= 2.1.3)
|
57
|
+
nokogiri (~> 1.5.0)
|
58
|
+
ruby-hmac
|
59
|
+
formatador (0.2.1)
|
47
60
|
i18n (0.5.0)
|
48
61
|
json (1.6.6)
|
49
62
|
mail (2.2.19)
|
@@ -52,6 +65,11 @@ GEM
|
|
52
65
|
mime-types (~> 1.16)
|
53
66
|
treetop (~> 1.4.8)
|
54
67
|
mime-types (1.18)
|
68
|
+
multi_json (1.2.0)
|
69
|
+
net-scp (1.0.4)
|
70
|
+
net-ssh (>= 1.99.1)
|
71
|
+
net-ssh (2.3.0)
|
72
|
+
nokogiri (1.5.2)
|
55
73
|
polyglot (0.3.3)
|
56
74
|
rack (1.2.5)
|
57
75
|
rack-mount (0.6.14)
|
@@ -83,6 +101,7 @@ GEM
|
|
83
101
|
rspec-expectations (2.9.1)
|
84
102
|
diff-lcs (~> 1.1.3)
|
85
103
|
rspec-mocks (2.9.0)
|
104
|
+
ruby-hmac (0.4.0)
|
86
105
|
sqlite3 (1.3.5)
|
87
106
|
thor (0.14.6)
|
88
107
|
treetop (1.4.10)
|
data/lib/beso/config.rb
CHANGED
@@ -2,23 +2,36 @@ module Beso
|
|
2
2
|
module Config
|
3
3
|
extend ActiveSupport::Concern
|
4
4
|
|
5
|
+
included do
|
6
|
+
reset!
|
7
|
+
end
|
8
|
+
|
5
9
|
module ClassMethods
|
6
10
|
def configure
|
7
11
|
yield self
|
8
12
|
end
|
9
13
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
mattr_accessor :access_key
|
15
|
+
mattr_accessor :secret_key
|
16
|
+
mattr_accessor :bucket_name
|
17
|
+
mattr_accessor :aws_region
|
18
|
+
|
19
|
+
def job( name, options, &block )
|
20
|
+
job = Job.new( name, options )
|
21
|
+
job.instance_eval &block if block_given?
|
22
|
+
jobs << job
|
23
|
+
end
|
24
|
+
|
25
|
+
def jobs
|
26
|
+
@@jobs ||= [ ]
|
18
27
|
end
|
19
28
|
|
20
|
-
def
|
21
|
-
|
29
|
+
def reset!
|
30
|
+
@@jobs = [ ]
|
31
|
+
@@access_key = nil
|
32
|
+
@@secret_key = nil
|
33
|
+
@@bucket_name = nil
|
34
|
+
@@aws_region = nil
|
22
35
|
end
|
23
36
|
end
|
24
37
|
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'fog'
|
2
|
+
|
3
|
+
module Beso
|
4
|
+
module Connection
|
5
|
+
|
6
|
+
class AWS
|
7
|
+
def initialize( options )
|
8
|
+
@access_key = options.delete( :access_key ) or raise MissingAccessKeyError
|
9
|
+
@secret_key = options.delete( :secret_key ) or raise MissingSecretKeyError
|
10
|
+
@bucket_name = options.delete( :bucket_name ) or raise MissingBucketNameError
|
11
|
+
@aws_region = options.delete( :aws_region )
|
12
|
+
end
|
13
|
+
|
14
|
+
def get( filename )
|
15
|
+
bucket.files.get filename
|
16
|
+
end
|
17
|
+
|
18
|
+
def read( filename )
|
19
|
+
get( filename ).try( :body ) || ''
|
20
|
+
end
|
21
|
+
|
22
|
+
def write( filename, body )
|
23
|
+
if file = get( filename )
|
24
|
+
file.body = body
|
25
|
+
file.save
|
26
|
+
else
|
27
|
+
storage.put_object @bucket_name, filename, body, headers
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def headers
|
34
|
+
{ 'x-amz-acl' => 'public-read' }
|
35
|
+
end
|
36
|
+
|
37
|
+
def bucket
|
38
|
+
@bucket ||= storage.directories.get @bucket_name
|
39
|
+
end
|
40
|
+
|
41
|
+
def storage
|
42
|
+
@storage ||= begin
|
43
|
+
storage = Fog::Storage.new :provider => 'AWS',
|
44
|
+
:aws_access_key_id => @access_key,
|
45
|
+
:aws_secret_access_key => @secret_key,
|
46
|
+
:region => @aws_region
|
47
|
+
storage.sync_clock
|
48
|
+
storage
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
extend ActiveSupport::Concern
|
54
|
+
|
55
|
+
module ClassMethods
|
56
|
+
|
57
|
+
def connect( &block )
|
58
|
+
yield AWS.new :access_key => Beso.access_key,
|
59
|
+
:secret_key => Beso.secret_key,
|
60
|
+
:bucket_name => Beso.bucket_name,
|
61
|
+
:aws_region => Beso.aws_region
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/beso/csv.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
module Beso
|
2
|
+
class CSV
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def method_missing( method, *args, &block )
|
6
|
+
csv.send method, *args, &block
|
7
|
+
end
|
8
|
+
|
9
|
+
protected
|
10
|
+
|
11
|
+
def csv
|
12
|
+
@csv ||= if require 'csv'
|
13
|
+
::CSV
|
14
|
+
else
|
15
|
+
require 'fastercsv'
|
16
|
+
FasterCSV
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/beso/job.rb
CHANGED
@@ -1,51 +1,94 @@
|
|
1
|
-
require 'comma'
|
2
|
-
|
3
1
|
module Beso
|
4
2
|
class Job
|
5
|
-
def initialize(
|
6
|
-
@
|
3
|
+
def initialize( event, options )
|
4
|
+
@event = event.to_sym
|
7
5
|
@table = options.delete :table
|
6
|
+
@since = options.delete :since
|
8
7
|
@props = { }
|
8
|
+
@extra = options
|
9
9
|
end
|
10
|
+
attr_reader :event
|
10
11
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
14
|
-
args[ 0 ] = "Prop:#{args[ 0 ] || sym.to_s.titleize}"
|
12
|
+
def identity( value=nil, &block )
|
13
|
+
@identity = value || block
|
14
|
+
end
|
15
15
|
|
16
|
-
|
16
|
+
def timestamp( value )
|
17
|
+
raise InvalidTimestampError unless value.is_a?(Symbol)
|
18
|
+
@timestamp = value
|
17
19
|
end
|
18
20
|
|
19
|
-
def
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def prop( name, value=nil, &block )
|
22
|
+
raise TooManyPropertiesError if @props.length == 10
|
23
|
+
@props[ name.to_sym ] = value || block || name.to_sym
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_csv( options={} )
|
27
|
+
raise MissingIdentityError if @identity.nil?
|
28
|
+
raise MissingTimestampError if @timestamp.nil?
|
29
|
+
|
30
|
+
@since ||= options.delete :since
|
23
31
|
|
24
|
-
model_class.
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
self.send sym, *args, &block
|
34
|
-
end
|
32
|
+
relation = model_class.where( "#{@timestamp} >= ?", @since || first_timestamp )
|
33
|
+
|
34
|
+
return nil if relation.empty?
|
35
|
+
|
36
|
+
Beso::CSV.generate( @extra.merge( options ) ) do |csv|
|
37
|
+
csv << ( required_headers + custom_headers )
|
38
|
+
|
39
|
+
relation.each do |model|
|
40
|
+
csv << ( required_columns( model ) + custom_columns( model ) )
|
35
41
|
end
|
36
42
|
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def first_timestamp
|
46
|
+
model_class.minimum @timestamp
|
47
|
+
end
|
37
48
|
|
38
|
-
|
49
|
+
def last_timestamp
|
50
|
+
model_class.maximum @timestamp
|
39
51
|
end
|
40
52
|
|
41
53
|
protected
|
42
54
|
|
55
|
+
def required_headers
|
56
|
+
%w| Identity Timestamp Event |
|
57
|
+
end
|
58
|
+
|
59
|
+
def custom_headers
|
60
|
+
@props.keys.map { |name| "Prop:#{name.to_s.titleize}" }
|
61
|
+
end
|
62
|
+
|
63
|
+
def required_columns( model )
|
64
|
+
[ ].tap do |row|
|
65
|
+
row << block_or_value( @identity, model )
|
66
|
+
row << model.send( @timestamp ).to_i
|
67
|
+
row << event_title
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def custom_columns( model )
|
72
|
+
@props.values.map { |value| block_or_value( value, model ) }
|
73
|
+
end
|
74
|
+
|
43
75
|
def event_title
|
44
|
-
@
|
76
|
+
@event.to_s.titleize
|
45
77
|
end
|
46
78
|
|
47
79
|
def model_class
|
48
80
|
@table.to_s.classify.constantize
|
49
81
|
end
|
82
|
+
|
83
|
+
def block_or_value( value, model )
|
84
|
+
case value
|
85
|
+
when Symbol
|
86
|
+
model.send value
|
87
|
+
when Proc
|
88
|
+
value.call model
|
89
|
+
else
|
90
|
+
value
|
91
|
+
end
|
92
|
+
end
|
50
93
|
end
|
51
94
|
end
|
data/lib/beso/railtie.rb
CHANGED
data/lib/beso/version.rb
CHANGED
data/lib/beso.rb
CHANGED
@@ -1,11 +1,23 @@
|
|
1
1
|
require 'beso/version'
|
2
2
|
|
3
3
|
module Beso
|
4
|
-
autoload :Config,
|
5
|
-
autoload :
|
6
|
-
autoload :
|
4
|
+
autoload :Config, 'beso/config'
|
5
|
+
autoload :Connection, 'beso/connection'
|
6
|
+
autoload :CSV, 'beso/csv'
|
7
|
+
autoload :Job, 'beso/job'
|
8
|
+
autoload :Railtie, 'beso/railtie'
|
7
9
|
|
8
10
|
include Config
|
11
|
+
include Connection
|
12
|
+
|
13
|
+
class BesoError < StandardError; end
|
14
|
+
class MissingIdentityError < BesoError; end
|
15
|
+
class MissingTimestampError < BesoError; end
|
16
|
+
class MissingAccessKeyError < BesoError; end
|
17
|
+
class InvalidTimestampError < BesoError; end
|
18
|
+
class MissingSecretKeyError < BesoError; end
|
19
|
+
class MissingBucketNameError < BesoError; end
|
20
|
+
class TooManyPropertiesError < BesoError; end
|
9
21
|
end
|
10
22
|
|
11
23
|
require 'beso/railtie' if defined?(Rails)
|
data/lib/tasks/beso.rake
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
namespace :beso do
|
2
|
+
|
3
|
+
desc "Run Beso jobs and upload to S3"
|
4
|
+
task :run => :environment do
|
5
|
+
|
6
|
+
raise 'Beso has no jobs to run. Please define some in the beso initializer.' if Beso.jobs.empty?
|
7
|
+
|
8
|
+
puts '==> Connecting...'
|
9
|
+
|
10
|
+
Beso.connect do |bucket|
|
11
|
+
|
12
|
+
puts '==> Connected!'
|
13
|
+
|
14
|
+
prefix = ENV[ 'BESO_PREFIX' ] || 'beso'
|
15
|
+
config = YAML.load( bucket.read 'beso.yml' ) || { }
|
16
|
+
|
17
|
+
puts '==> Config:'
|
18
|
+
puts config.inspect
|
19
|
+
|
20
|
+
Beso.jobs.each do |job|
|
21
|
+
|
22
|
+
config[ job.event ] ||= job.first_timestamp
|
23
|
+
|
24
|
+
puts "==> Processing job: #{job.event.inspect} since #{config[ job.event ]}"
|
25
|
+
|
26
|
+
csv = job.to_csv :since => config[ job.event ]
|
27
|
+
|
28
|
+
if csv
|
29
|
+
filename = "#{prefix}-#{job.event}-#{config[ job.event ].to_i}.csv"
|
30
|
+
|
31
|
+
bucket.write filename, csv
|
32
|
+
|
33
|
+
puts " ==> #{filename} saved to S3"
|
34
|
+
|
35
|
+
config[ job.event ] = job.last_timestamp
|
36
|
+
|
37
|
+
puts " ==> New timestamp is #{config[ job.event ]}"
|
38
|
+
else
|
39
|
+
puts " ==> No records found since #{config[ job.event ]}. Skipping..."
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
bucket.write 'beso.yml', config.to_yaml
|
44
|
+
|
45
|
+
puts '==> Config saved. Donezo.'
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Beso::Connection::AWS do
|
4
|
+
|
5
|
+
context 'without an access key' do
|
6
|
+
it 'should raise an error' do
|
7
|
+
expect { Beso::AWS.new :secret_key => 'foo' }.to raise_error Beso::MissingAccessKeyError
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'without a secret key' do
|
12
|
+
it 'should raise an error' do
|
13
|
+
expect { Beso::AWS.new :access_key => 'foo' }.to raise_error Beso::MissingSecretKeyError
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/spec/beso/config_spec.rb
CHANGED
@@ -13,39 +13,76 @@ describe Beso do
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
describe '#
|
16
|
+
describe '#job' do
|
17
|
+
subject { Beso }
|
18
|
+
|
17
19
|
before do
|
18
|
-
|
20
|
+
yielded = nil
|
21
|
+
subject.job :foo, :table => :users do
|
22
|
+
yielded = self
|
23
|
+
end
|
24
|
+
yielded.should be_a( Beso::Job )
|
19
25
|
end
|
20
26
|
|
21
|
-
|
27
|
+
it { should have( 1 ).jobs }
|
28
|
+
end
|
22
29
|
|
23
|
-
|
24
|
-
|
25
|
-
|
30
|
+
describe '#reset' do
|
31
|
+
subject { Beso }
|
32
|
+
it { should respond_to( :reset! ) }
|
26
33
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
end
|
34
|
+
it 'should reset the #jobs array' do
|
35
|
+
Beso.jobs << 123
|
36
|
+
Beso.reset!
|
37
|
+
Beso.jobs.should be_empty
|
38
|
+
end
|
33
39
|
|
34
|
-
|
40
|
+
it 'should reset the #access_key' do
|
41
|
+
Beso.access_key = 'foo'
|
42
|
+
Beso.reset!
|
43
|
+
Beso.access_key.should be_nil
|
35
44
|
end
|
36
45
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
46
|
+
it 'should reset the #secret_key' do
|
47
|
+
Beso.secret_key = 'foo'
|
48
|
+
Beso.reset!
|
49
|
+
Beso.secret_key.should be_nil
|
50
|
+
end
|
43
51
|
|
44
|
-
|
45
|
-
|
46
|
-
|
52
|
+
it 'should reset the #bucket_name' do
|
53
|
+
Beso.bucket_name = 'beso'
|
54
|
+
Beso.reset!
|
55
|
+
Beso.bucket_name.should be_nil
|
56
|
+
end
|
47
57
|
|
48
|
-
|
58
|
+
it 'should reset the #aws_region' do
|
59
|
+
Beso.aws_region = 'us-east-1'
|
60
|
+
Beso.reset!
|
61
|
+
Beso.aws_region.should be_nil
|
49
62
|
end
|
50
63
|
end
|
64
|
+
|
65
|
+
describe '#access_key' do
|
66
|
+
subject { Beso }
|
67
|
+
it { should respond_to( :access_key ) }
|
68
|
+
its( :access_key ){ should be_nil }
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#secret_key' do
|
72
|
+
subject { Beso }
|
73
|
+
it { should respond_to( :secret_key ) }
|
74
|
+
its( :secret_key ){ should be_nil }
|
75
|
+
end
|
76
|
+
|
77
|
+
describe '#bucket_name' do
|
78
|
+
subject { Beso }
|
79
|
+
it { should respond_to( :bucket_name ) }
|
80
|
+
its( :bucket_name ){ should be_nil}
|
81
|
+
end
|
82
|
+
|
83
|
+
describe '#aws_region' do
|
84
|
+
subject { Beso }
|
85
|
+
it { should respond_to( :aws_region ) }
|
86
|
+
its( :aws_region ){ should be_nil }
|
87
|
+
end
|
51
88
|
end
|
data/spec/beso/job_spec.rb
CHANGED
@@ -2,17 +2,46 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe Beso::Job do
|
4
4
|
|
5
|
+
before do
|
6
|
+
User.destroy_all
|
7
|
+
end
|
8
|
+
|
5
9
|
describe 'to_csv' do
|
6
10
|
subject { Beso::Job.new :message_sent, :table => :users }
|
7
11
|
|
8
|
-
before do
|
9
|
-
User.destroy_all
|
10
|
-
end
|
11
|
-
|
12
12
|
let!( :foo ){ User.create! :name => 'Foo' }
|
13
13
|
let!( :bar ){ User.create! :name => 'Bar' }
|
14
14
|
|
15
|
-
context '
|
15
|
+
context 'without an identity defined' do
|
16
|
+
before do
|
17
|
+
subject.timestamp :created_at
|
18
|
+
end
|
19
|
+
it 'should raise an error' do
|
20
|
+
expect { subject.to_csv }.to raise_error( Beso::MissingIdentityError )
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'without a timestamp defined' do
|
25
|
+
before do
|
26
|
+
subject.identity { |user| user.id }
|
27
|
+
end
|
28
|
+
it 'should raise an error' do
|
29
|
+
expect { subject.to_csv }.to raise_error( Beso::MissingTimestampError )
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'with a timestamp that is not a symbol' do
|
34
|
+
it 'should raise an error' do
|
35
|
+
expect { subject.timestamp 123 }.to raise_error( Beso::InvalidTimestampError )
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'with only the mandatory columns defined' do
|
40
|
+
before do
|
41
|
+
subject.identity { |user| user.id }
|
42
|
+
subject.timestamp :created_at
|
43
|
+
end
|
44
|
+
|
16
45
|
its( :to_csv ){ should eq( <<-EOS
|
17
46
|
Identity,Timestamp,Event
|
18
47
|
#{foo.id},#{foo.created_at.to_i},Message Sent
|
@@ -21,61 +50,234 @@ Identity,Timestamp,Event
|
|
21
50
|
) }
|
22
51
|
end
|
23
52
|
|
24
|
-
context 'with
|
53
|
+
context 'with custom properties defined' do
|
25
54
|
before do
|
26
|
-
subject.
|
55
|
+
subject.identity { |user| user.id }
|
56
|
+
subject.timestamp :created_at
|
57
|
+
subject.prop( :user_name ){ |user| user.name }
|
27
58
|
end
|
28
59
|
|
29
60
|
its( :to_csv ){ should eq( <<-EOS
|
30
|
-
Identity,Timestamp,Event,Prop:Name
|
31
|
-
#{foo.id},#{foo.created_at.to_i},Message Sent
|
32
|
-
#{bar.id},#{bar.created_at.to_i},Message Sent
|
61
|
+
Identity,Timestamp,Event,Prop:User Name
|
62
|
+
#{foo.id},#{foo.created_at.to_i},Message Sent,Foo
|
63
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent,Bar
|
64
|
+
EOS
|
65
|
+
) }
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'with literal properties defined' do
|
69
|
+
before do
|
70
|
+
subject.identity 22
|
71
|
+
subject.timestamp :created_at
|
72
|
+
subject.prop :foo, 'bar'
|
73
|
+
end
|
74
|
+
|
75
|
+
its( :to_csv ){ should eq( <<-EOS
|
76
|
+
Identity,Timestamp,Event,Prop:Foo
|
77
|
+
22,#{foo.created_at.to_i},Message Sent,bar
|
78
|
+
22,#{bar.created_at.to_i},Message Sent,bar
|
33
79
|
EOS
|
34
80
|
) }
|
35
81
|
end
|
36
82
|
|
37
|
-
context 'with
|
83
|
+
context 'with symbol properties defined' do
|
38
84
|
before do
|
39
|
-
subject.
|
85
|
+
subject.identity :id
|
86
|
+
subject.timestamp :created_at
|
87
|
+
subject.prop :name
|
40
88
|
end
|
41
89
|
|
42
90
|
its( :to_csv ){ should eq( <<-EOS
|
43
|
-
Identity,Timestamp,Event,Prop:
|
91
|
+
Identity,Timestamp,Event,Prop:Name
|
44
92
|
#{foo.id},#{foo.created_at.to_i},Message Sent,#{foo.name}
|
45
93
|
#{bar.id},#{bar.created_at.to_i},Message Sent,#{bar.name}
|
46
94
|
EOS
|
47
95
|
) }
|
48
96
|
end
|
49
97
|
|
50
|
-
context 'with
|
98
|
+
context 'with 10 custom properties defined' do
|
51
99
|
before do
|
52
|
-
subject.
|
53
|
-
|
100
|
+
subject.identity 22
|
101
|
+
subject.timestamp :created_at
|
102
|
+
(1..10).each do |i|
|
103
|
+
subject.prop :"foo #{i}", i
|
54
104
|
end
|
55
105
|
end
|
56
106
|
|
57
107
|
its( :to_csv ){ should eq( <<-EOS
|
58
|
-
Identity,Timestamp,Event,Prop:
|
59
|
-
|
60
|
-
|
108
|
+
Identity,Timestamp,Event,Prop:Foo 1,Prop:Foo 2,Prop:Foo 3,Prop:Foo 4,Prop:Foo 5,Prop:Foo 6,Prop:Foo 7,Prop:Foo 8,Prop:Foo 9,Prop:Foo 10
|
109
|
+
22,#{foo.created_at.to_i},Message Sent,1,2,3,4,5,6,7,8,9,10
|
110
|
+
22,#{bar.created_at.to_i},Message Sent,1,2,3,4,5,6,7,8,9,10
|
61
111
|
EOS
|
62
|
-
)}
|
112
|
+
) }
|
63
113
|
end
|
64
114
|
|
65
|
-
context 'with
|
115
|
+
context 'with more than 10 custom properties defined' do
|
66
116
|
before do
|
67
|
-
subject.
|
68
|
-
|
69
|
-
|
117
|
+
subject.identity 22
|
118
|
+
subject.timestamp :created_at
|
119
|
+
end
|
120
|
+
it 'should raise an error' do
|
121
|
+
expect {
|
122
|
+
(1..11).each do |i|
|
123
|
+
subject.prop :"foo #{i}", i
|
124
|
+
end
|
125
|
+
}.to raise_error Beso::TooManyPropertiesError
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
context 'with custom options given to #to_csv' do
|
130
|
+
before do
|
131
|
+
subject.identity { |user| user.id }
|
132
|
+
subject.timestamp :created_at
|
133
|
+
end
|
134
|
+
|
135
|
+
it 'should support all options that CSV supports' do
|
136
|
+
subject.to_csv( :force_quotes => true, :col_sep => ';' ).should eq( <<-EOS
|
137
|
+
"Identity";"Timestamp";"Event"
|
138
|
+
"#{foo.id}";"#{foo.created_at.to_i}";"Message Sent"
|
139
|
+
"#{bar.id}";"#{bar.created_at.to_i}";"Message Sent"
|
140
|
+
EOS
|
141
|
+
)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
context 'with custom options given to constructor' do
|
146
|
+
subject { Beso::Job.new :message_sent, :table => :users, :col_sep => ';' }
|
147
|
+
|
148
|
+
before do
|
149
|
+
subject.identity { |user| user.id }
|
150
|
+
subject.timestamp :created_at
|
70
151
|
end
|
71
152
|
|
72
153
|
its( :to_csv ){ should eq( <<-EOS
|
73
|
-
Identity
|
74
|
-
#{foo.id}
|
75
|
-
#{bar.id}
|
154
|
+
Identity;Timestamp;Event
|
155
|
+
#{foo.id};#{foo.created_at.to_i};Message Sent
|
156
|
+
#{bar.id};#{bar.created_at.to_i};Message Sent
|
76
157
|
EOS
|
77
|
-
)}
|
158
|
+
) }
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'when no records match the query' do
|
162
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
163
|
+
|
164
|
+
before do
|
165
|
+
User.destroy_all
|
166
|
+
subject.identity { |user| user.id }
|
167
|
+
subject.timestamp :created_at
|
168
|
+
end
|
169
|
+
|
170
|
+
its( :to_csv ){ should be_nil }
|
78
171
|
end
|
79
172
|
end
|
80
173
|
|
174
|
+
describe 'with since specified' do
|
175
|
+
let!( :foo ){ User.create :name => 'Foo', :created_at => 100, :updated_at => 300 }
|
176
|
+
let!( :bar ){ User.create :name => 'Bar', :created_at => 200, :updated_at => 200 }
|
177
|
+
let!( :baz ){ User.create :name => 'Baz', :created_at => 300, :updated_at => 300 }
|
178
|
+
|
179
|
+
context 'in the constructor' do
|
180
|
+
context 'and the timestamp keyed on `created_at`' do
|
181
|
+
subject { Beso::Job.new :message_sent, :table => :users, :since => 101 }
|
182
|
+
|
183
|
+
before do
|
184
|
+
subject.identity { |user| user.id }
|
185
|
+
subject.timestamp :created_at
|
186
|
+
end
|
187
|
+
|
188
|
+
its( :to_csv ){ should eq( <<-EOS
|
189
|
+
Identity,Timestamp,Event
|
190
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent
|
191
|
+
#{baz.id},#{baz.created_at.to_i},Message Sent
|
192
|
+
EOS
|
193
|
+
) }
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'in to_csv' do
|
198
|
+
context 'and the timestamp keyed on `created_at`' do
|
199
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
200
|
+
|
201
|
+
before do
|
202
|
+
subject.identity { |user| user.id }
|
203
|
+
subject.timestamp :created_at
|
204
|
+
end
|
205
|
+
|
206
|
+
it 'should find records after `since`' do
|
207
|
+
subject.to_csv( :since => 101 ).should eq( <<-EOS
|
208
|
+
Identity,Timestamp,Event
|
209
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent
|
210
|
+
#{baz.id},#{baz.created_at.to_i},Message Sent
|
211
|
+
EOS
|
212
|
+
)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
context 'in the constructor' do
|
218
|
+
context 'and the timestamp keyed on `updated_at`' do
|
219
|
+
subject { Beso::Job.new :message_sent, :table => :users, :since => 201 }
|
220
|
+
|
221
|
+
before do
|
222
|
+
subject.identity { |user| user.id }
|
223
|
+
subject.timestamp :updated_at
|
224
|
+
end
|
225
|
+
|
226
|
+
its( :to_csv ){ should eq( <<-EOS
|
227
|
+
Identity,Timestamp,Event
|
228
|
+
#{foo.id},#{foo.updated_at.to_i},Message Sent
|
229
|
+
#{baz.id},#{baz.updated_at.to_i},Message Sent
|
230
|
+
EOS
|
231
|
+
) }
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
context 'in to_csv' do
|
236
|
+
context 'and the timestamp keyed on `updated_at`' do
|
237
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
238
|
+
|
239
|
+
before do
|
240
|
+
subject.identity { |user| user.id }
|
241
|
+
subject.timestamp :updated_at
|
242
|
+
end
|
243
|
+
|
244
|
+
it 'should find records after `since`' do
|
245
|
+
subject.to_csv( :since => 201 ).should eq( <<-EOS
|
246
|
+
Identity,Timestamp,Event
|
247
|
+
#{foo.id},#{foo.updated_at.to_i},Message Sent
|
248
|
+
#{baz.id},#{baz.updated_at.to_i},Message Sent
|
249
|
+
EOS
|
250
|
+
)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
describe '#last_timestamp' do
|
257
|
+
let!( :foo ){ User.create :name => 'Foo', :created_at => 100, :updated_at => 301 }
|
258
|
+
let!( :bar ){ User.create :name => 'Bar', :created_at => 200, :updated_at => 200 }
|
259
|
+
let!( :baz ){ User.create :name => 'Baz', :created_at => 300, :updated_at => 300 }
|
260
|
+
|
261
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
262
|
+
|
263
|
+
before do
|
264
|
+
subject.identity :id
|
265
|
+
end
|
266
|
+
|
267
|
+
context 'with the timestamp keyed on `created_at`' do
|
268
|
+
before do
|
269
|
+
subject.timestamp :created_at
|
270
|
+
end
|
271
|
+
|
272
|
+
its( :last_timestamp ){ should eq( 300 ) }
|
273
|
+
end
|
274
|
+
|
275
|
+
context 'with the timestamp keyed on `updated_at`' do
|
276
|
+
before do
|
277
|
+
subject.timestamp :updated_at
|
278
|
+
end
|
279
|
+
|
280
|
+
its( :last_timestamp ){ should eq( 301 ) }
|
281
|
+
end
|
282
|
+
end
|
81
283
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beso
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-04-
|
12
|
+
date: 2012-04-12 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -28,13 +28,13 @@ dependencies:
|
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 3.0.10
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: fastercsv
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
35
|
- - ! '>='
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version:
|
37
|
+
version: 1.5.4
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,23 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
45
|
+
version: 1.5.4
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: fog
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.3.1
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.3.1
|
46
62
|
- !ruby/object:Gem::Dependency
|
47
63
|
name: sqlite3
|
48
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,12 +123,15 @@ files:
|
|
107
123
|
- beso.gemspec
|
108
124
|
- gemfiles/rails-3.0.10.gemfile
|
109
125
|
- gemfiles/rails-3.0.10.gemfile.lock
|
110
|
-
- lib/beso.rake
|
111
126
|
- lib/beso.rb
|
112
127
|
- lib/beso/config.rb
|
128
|
+
- lib/beso/connection.rb
|
129
|
+
- lib/beso/csv.rb
|
113
130
|
- lib/beso/job.rb
|
114
131
|
- lib/beso/railtie.rb
|
115
132
|
- lib/beso/version.rb
|
133
|
+
- lib/tasks/beso.rake
|
134
|
+
- spec/beso/aws_spec.rb
|
116
135
|
- spec/beso/config_spec.rb
|
117
136
|
- spec/beso/job_spec.rb
|
118
137
|
- spec/rails/.gitkeep
|
@@ -143,6 +162,7 @@ signing_key:
|
|
143
162
|
specification_version: 3
|
144
163
|
summary: Sync your KISSmetrics history, guapo!
|
145
164
|
test_files:
|
165
|
+
- spec/beso/aws_spec.rb
|
146
166
|
- spec/beso/config_spec.rb
|
147
167
|
- spec/beso/job_spec.rb
|
148
168
|
- spec/rails/.gitkeep
|