beso 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/beso.gemspec +2 -1
- data/gemfiles/rails-3.0.10.gemfile.lock +21 -2
- data/lib/beso/config.rb +23 -10
- data/lib/beso/connection.rb +65 -0
- data/lib/beso/csv.rb +22 -0
- data/lib/beso/job.rb +69 -26
- data/lib/beso/railtie.rb +1 -1
- data/lib/beso/version.rb +1 -1
- data/lib/beso.rb +15 -3
- data/lib/tasks/beso.rake +48 -0
- data/spec/beso/aws_spec.rb +16 -0
- data/spec/beso/config_spec.rb +60 -23
- data/spec/beso/job_spec.rb +230 -28
- data/spec/spec_helper.rb +4 -0
- metadata +26 -6
- data/lib/beso.rake +0 -6
data/beso.gemspec
CHANGED
@@ -16,7 +16,8 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.version = Beso::VERSION
|
17
17
|
|
18
18
|
gem.add_dependency 'rails', '>= 3.0.10'
|
19
|
-
gem.add_dependency '
|
19
|
+
gem.add_dependency 'fastercsv', '>= 1.5.4'
|
20
|
+
gem.add_dependency 'fog', '>= 1.3.1'
|
20
21
|
|
21
22
|
gem.add_development_dependency 'sqlite3'
|
22
23
|
gem.add_development_dependency 'appraisal', '>= 0.4.1'
|
@@ -2,7 +2,8 @@ PATH
|
|
2
2
|
remote: /Users/jeremyruppel/Git/beso
|
3
3
|
specs:
|
4
4
|
beso (0.0.1)
|
5
|
-
|
5
|
+
fastercsv (>= 1.5.4)
|
6
|
+
fog (>= 1.3.1)
|
6
7
|
rails (>= 3.0.10)
|
7
8
|
|
8
9
|
GEM
|
@@ -40,10 +41,22 @@ GEM
|
|
40
41
|
rake
|
41
42
|
arel (2.0.10)
|
42
43
|
builder (2.1.2)
|
43
|
-
comma (3.0.3)
|
44
44
|
diff-lcs (1.1.3)
|
45
45
|
erubis (2.6.6)
|
46
46
|
abstract (>= 1.0.0)
|
47
|
+
excon (0.13.3)
|
48
|
+
fastercsv (1.5.4)
|
49
|
+
fog (1.3.1)
|
50
|
+
builder
|
51
|
+
excon (~> 0.13.0)
|
52
|
+
formatador (~> 0.2.0)
|
53
|
+
mime-types
|
54
|
+
multi_json (~> 1.0)
|
55
|
+
net-scp (~> 1.0.4)
|
56
|
+
net-ssh (>= 2.1.3)
|
57
|
+
nokogiri (~> 1.5.0)
|
58
|
+
ruby-hmac
|
59
|
+
formatador (0.2.1)
|
47
60
|
i18n (0.5.0)
|
48
61
|
json (1.6.6)
|
49
62
|
mail (2.2.19)
|
@@ -52,6 +65,11 @@ GEM
|
|
52
65
|
mime-types (~> 1.16)
|
53
66
|
treetop (~> 1.4.8)
|
54
67
|
mime-types (1.18)
|
68
|
+
multi_json (1.2.0)
|
69
|
+
net-scp (1.0.4)
|
70
|
+
net-ssh (>= 1.99.1)
|
71
|
+
net-ssh (2.3.0)
|
72
|
+
nokogiri (1.5.2)
|
55
73
|
polyglot (0.3.3)
|
56
74
|
rack (1.2.5)
|
57
75
|
rack-mount (0.6.14)
|
@@ -83,6 +101,7 @@ GEM
|
|
83
101
|
rspec-expectations (2.9.1)
|
84
102
|
diff-lcs (~> 1.1.3)
|
85
103
|
rspec-mocks (2.9.0)
|
104
|
+
ruby-hmac (0.4.0)
|
86
105
|
sqlite3 (1.3.5)
|
87
106
|
thor (0.14.6)
|
88
107
|
treetop (1.4.10)
|
data/lib/beso/config.rb
CHANGED
@@ -2,23 +2,36 @@ module Beso
|
|
2
2
|
module Config
|
3
3
|
extend ActiveSupport::Concern
|
4
4
|
|
5
|
+
included do
|
6
|
+
reset!
|
7
|
+
end
|
8
|
+
|
5
9
|
module ClassMethods
|
6
10
|
def configure
|
7
11
|
yield self
|
8
12
|
end
|
9
13
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
mattr_accessor :access_key
|
15
|
+
mattr_accessor :secret_key
|
16
|
+
mattr_accessor :bucket_name
|
17
|
+
mattr_accessor :aws_region
|
18
|
+
|
19
|
+
def job( name, options, &block )
|
20
|
+
job = Job.new( name, options )
|
21
|
+
job.instance_eval &block if block_given?
|
22
|
+
jobs << job
|
23
|
+
end
|
24
|
+
|
25
|
+
def jobs
|
26
|
+
@@jobs ||= [ ]
|
18
27
|
end
|
19
28
|
|
20
|
-
def
|
21
|
-
|
29
|
+
def reset!
|
30
|
+
@@jobs = [ ]
|
31
|
+
@@access_key = nil
|
32
|
+
@@secret_key = nil
|
33
|
+
@@bucket_name = nil
|
34
|
+
@@aws_region = nil
|
22
35
|
end
|
23
36
|
end
|
24
37
|
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'fog'
|
2
|
+
|
3
|
+
module Beso
|
4
|
+
module Connection
|
5
|
+
|
6
|
+
class AWS
|
7
|
+
def initialize( options )
|
8
|
+
@access_key = options.delete( :access_key ) or raise MissingAccessKeyError
|
9
|
+
@secret_key = options.delete( :secret_key ) or raise MissingSecretKeyError
|
10
|
+
@bucket_name = options.delete( :bucket_name ) or raise MissingBucketNameError
|
11
|
+
@aws_region = options.delete( :aws_region )
|
12
|
+
end
|
13
|
+
|
14
|
+
def get( filename )
|
15
|
+
bucket.files.get filename
|
16
|
+
end
|
17
|
+
|
18
|
+
def read( filename )
|
19
|
+
get( filename ).try( :body ) || ''
|
20
|
+
end
|
21
|
+
|
22
|
+
def write( filename, body )
|
23
|
+
if file = get( filename )
|
24
|
+
file.body = body
|
25
|
+
file.save
|
26
|
+
else
|
27
|
+
storage.put_object @bucket_name, filename, body, headers
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def headers
|
34
|
+
{ 'x-amz-acl' => 'public-read' }
|
35
|
+
end
|
36
|
+
|
37
|
+
def bucket
|
38
|
+
@bucket ||= storage.directories.get @bucket_name
|
39
|
+
end
|
40
|
+
|
41
|
+
def storage
|
42
|
+
@storage ||= begin
|
43
|
+
storage = Fog::Storage.new :provider => 'AWS',
|
44
|
+
:aws_access_key_id => @access_key,
|
45
|
+
:aws_secret_access_key => @secret_key,
|
46
|
+
:region => @aws_region
|
47
|
+
storage.sync_clock
|
48
|
+
storage
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
extend ActiveSupport::Concern
|
54
|
+
|
55
|
+
module ClassMethods
|
56
|
+
|
57
|
+
def connect( &block )
|
58
|
+
yield AWS.new :access_key => Beso.access_key,
|
59
|
+
:secret_key => Beso.secret_key,
|
60
|
+
:bucket_name => Beso.bucket_name,
|
61
|
+
:aws_region => Beso.aws_region
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/beso/csv.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
module Beso
|
2
|
+
class CSV
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def method_missing( method, *args, &block )
|
6
|
+
csv.send method, *args, &block
|
7
|
+
end
|
8
|
+
|
9
|
+
protected
|
10
|
+
|
11
|
+
def csv
|
12
|
+
@csv ||= if require 'csv'
|
13
|
+
::CSV
|
14
|
+
else
|
15
|
+
require 'fastercsv'
|
16
|
+
FasterCSV
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/beso/job.rb
CHANGED
@@ -1,51 +1,94 @@
|
|
1
|
-
require 'comma'
|
2
|
-
|
3
1
|
module Beso
|
4
2
|
class Job
|
5
|
-
def initialize(
|
6
|
-
@
|
3
|
+
def initialize( event, options )
|
4
|
+
@event = event.to_sym
|
7
5
|
@table = options.delete :table
|
6
|
+
@since = options.delete :since
|
8
7
|
@props = { }
|
8
|
+
@extra = options
|
9
9
|
end
|
10
|
+
attr_reader :event
|
10
11
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
14
|
-
args[ 0 ] = "Prop:#{args[ 0 ] || sym.to_s.titleize}"
|
12
|
+
def identity( value=nil, &block )
|
13
|
+
@identity = value || block
|
14
|
+
end
|
15
15
|
|
16
|
-
|
16
|
+
def timestamp( value )
|
17
|
+
raise InvalidTimestampError unless value.is_a?(Symbol)
|
18
|
+
@timestamp = value
|
17
19
|
end
|
18
20
|
|
19
|
-
def
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def prop( name, value=nil, &block )
|
22
|
+
raise TooManyPropertiesError if @props.length == 10
|
23
|
+
@props[ name.to_sym ] = value || block || name.to_sym
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_csv( options={} )
|
27
|
+
raise MissingIdentityError if @identity.nil?
|
28
|
+
raise MissingTimestampError if @timestamp.nil?
|
29
|
+
|
30
|
+
@since ||= options.delete :since
|
23
31
|
|
24
|
-
model_class.
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
self.send sym, *args, &block
|
34
|
-
end
|
32
|
+
relation = model_class.where( "#{@timestamp} >= ?", @since || first_timestamp )
|
33
|
+
|
34
|
+
return nil if relation.empty?
|
35
|
+
|
36
|
+
Beso::CSV.generate( @extra.merge( options ) ) do |csv|
|
37
|
+
csv << ( required_headers + custom_headers )
|
38
|
+
|
39
|
+
relation.each do |model|
|
40
|
+
csv << ( required_columns( model ) + custom_columns( model ) )
|
35
41
|
end
|
36
42
|
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def first_timestamp
|
46
|
+
model_class.minimum @timestamp
|
47
|
+
end
|
37
48
|
|
38
|
-
|
49
|
+
def last_timestamp
|
50
|
+
model_class.maximum @timestamp
|
39
51
|
end
|
40
52
|
|
41
53
|
protected
|
42
54
|
|
55
|
+
def required_headers
|
56
|
+
%w| Identity Timestamp Event |
|
57
|
+
end
|
58
|
+
|
59
|
+
def custom_headers
|
60
|
+
@props.keys.map { |name| "Prop:#{name.to_s.titleize}" }
|
61
|
+
end
|
62
|
+
|
63
|
+
def required_columns( model )
|
64
|
+
[ ].tap do |row|
|
65
|
+
row << block_or_value( @identity, model )
|
66
|
+
row << model.send( @timestamp ).to_i
|
67
|
+
row << event_title
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def custom_columns( model )
|
72
|
+
@props.values.map { |value| block_or_value( value, model ) }
|
73
|
+
end
|
74
|
+
|
43
75
|
def event_title
|
44
|
-
@
|
76
|
+
@event.to_s.titleize
|
45
77
|
end
|
46
78
|
|
47
79
|
def model_class
|
48
80
|
@table.to_s.classify.constantize
|
49
81
|
end
|
82
|
+
|
83
|
+
def block_or_value( value, model )
|
84
|
+
case value
|
85
|
+
when Symbol
|
86
|
+
model.send value
|
87
|
+
when Proc
|
88
|
+
value.call model
|
89
|
+
else
|
90
|
+
value
|
91
|
+
end
|
92
|
+
end
|
50
93
|
end
|
51
94
|
end
|
data/lib/beso/railtie.rb
CHANGED
data/lib/beso/version.rb
CHANGED
data/lib/beso.rb
CHANGED
@@ -1,11 +1,23 @@
|
|
1
1
|
require 'beso/version'
|
2
2
|
|
3
3
|
module Beso
|
4
|
-
autoload :Config,
|
5
|
-
autoload :
|
6
|
-
autoload :
|
4
|
+
autoload :Config, 'beso/config'
|
5
|
+
autoload :Connection, 'beso/connection'
|
6
|
+
autoload :CSV, 'beso/csv'
|
7
|
+
autoload :Job, 'beso/job'
|
8
|
+
autoload :Railtie, 'beso/railtie'
|
7
9
|
|
8
10
|
include Config
|
11
|
+
include Connection
|
12
|
+
|
13
|
+
class BesoError < StandardError; end
|
14
|
+
class MissingIdentityError < BesoError; end
|
15
|
+
class MissingTimestampError < BesoError; end
|
16
|
+
class MissingAccessKeyError < BesoError; end
|
17
|
+
class InvalidTimestampError < BesoError; end
|
18
|
+
class MissingSecretKeyError < BesoError; end
|
19
|
+
class MissingBucketNameError < BesoError; end
|
20
|
+
class TooManyPropertiesError < BesoError; end
|
9
21
|
end
|
10
22
|
|
11
23
|
require 'beso/railtie' if defined?(Rails)
|
data/lib/tasks/beso.rake
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
namespace :beso do
|
2
|
+
|
3
|
+
desc "Run Beso jobs and upload to S3"
|
4
|
+
task :run => :environment do
|
5
|
+
|
6
|
+
raise 'Beso has no jobs to run. Please define some in the beso initializer.' if Beso.jobs.empty?
|
7
|
+
|
8
|
+
puts '==> Connecting...'
|
9
|
+
|
10
|
+
Beso.connect do |bucket|
|
11
|
+
|
12
|
+
puts '==> Connected!'
|
13
|
+
|
14
|
+
prefix = ENV[ 'BESO_PREFIX' ] || 'beso'
|
15
|
+
config = YAML.load( bucket.read 'beso.yml' ) || { }
|
16
|
+
|
17
|
+
puts '==> Config:'
|
18
|
+
puts config.inspect
|
19
|
+
|
20
|
+
Beso.jobs.each do |job|
|
21
|
+
|
22
|
+
config[ job.event ] ||= job.first_timestamp
|
23
|
+
|
24
|
+
puts "==> Processing job: #{job.event.inspect} since #{config[ job.event ]}"
|
25
|
+
|
26
|
+
csv = job.to_csv :since => config[ job.event ]
|
27
|
+
|
28
|
+
if csv
|
29
|
+
filename = "#{prefix}-#{job.event}-#{config[ job.event ].to_i}.csv"
|
30
|
+
|
31
|
+
bucket.write filename, csv
|
32
|
+
|
33
|
+
puts " ==> #{filename} saved to S3"
|
34
|
+
|
35
|
+
config[ job.event ] = job.last_timestamp
|
36
|
+
|
37
|
+
puts " ==> New timestamp is #{config[ job.event ]}"
|
38
|
+
else
|
39
|
+
puts " ==> No records found since #{config[ job.event ]}. Skipping..."
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
bucket.write 'beso.yml', config.to_yaml
|
44
|
+
|
45
|
+
puts '==> Config saved. Donezo.'
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Beso::Connection::AWS do
|
4
|
+
|
5
|
+
context 'without an access key' do
|
6
|
+
it 'should raise an error' do
|
7
|
+
expect { Beso::AWS.new :secret_key => 'foo' }.to raise_error Beso::MissingAccessKeyError
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
context 'without a secret key' do
|
12
|
+
it 'should raise an error' do
|
13
|
+
expect { Beso::AWS.new :access_key => 'foo' }.to raise_error Beso::MissingSecretKeyError
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/spec/beso/config_spec.rb
CHANGED
@@ -13,39 +13,76 @@ describe Beso do
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
describe '#
|
16
|
+
describe '#job' do
|
17
|
+
subject { Beso }
|
18
|
+
|
17
19
|
before do
|
18
|
-
|
20
|
+
yielded = nil
|
21
|
+
subject.job :foo, :table => :users do
|
22
|
+
yielded = self
|
23
|
+
end
|
24
|
+
yielded.should be_a( Beso::Job )
|
19
25
|
end
|
20
26
|
|
21
|
-
|
27
|
+
it { should have( 1 ).jobs }
|
28
|
+
end
|
22
29
|
|
23
|
-
|
24
|
-
|
25
|
-
|
30
|
+
describe '#reset' do
|
31
|
+
subject { Beso }
|
32
|
+
it { should respond_to( :reset! ) }
|
26
33
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
end
|
34
|
+
it 'should reset the #jobs array' do
|
35
|
+
Beso.jobs << 123
|
36
|
+
Beso.reset!
|
37
|
+
Beso.jobs.should be_empty
|
38
|
+
end
|
33
39
|
|
34
|
-
|
40
|
+
it 'should reset the #access_key' do
|
41
|
+
Beso.access_key = 'foo'
|
42
|
+
Beso.reset!
|
43
|
+
Beso.access_key.should be_nil
|
35
44
|
end
|
36
45
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
46
|
+
it 'should reset the #secret_key' do
|
47
|
+
Beso.secret_key = 'foo'
|
48
|
+
Beso.reset!
|
49
|
+
Beso.secret_key.should be_nil
|
50
|
+
end
|
43
51
|
|
44
|
-
|
45
|
-
|
46
|
-
|
52
|
+
it 'should reset the #bucket_name' do
|
53
|
+
Beso.bucket_name = 'beso'
|
54
|
+
Beso.reset!
|
55
|
+
Beso.bucket_name.should be_nil
|
56
|
+
end
|
47
57
|
|
48
|
-
|
58
|
+
it 'should reset the #aws_region' do
|
59
|
+
Beso.aws_region = 'us-east-1'
|
60
|
+
Beso.reset!
|
61
|
+
Beso.aws_region.should be_nil
|
49
62
|
end
|
50
63
|
end
|
64
|
+
|
65
|
+
describe '#access_key' do
|
66
|
+
subject { Beso }
|
67
|
+
it { should respond_to( :access_key ) }
|
68
|
+
its( :access_key ){ should be_nil }
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#secret_key' do
|
72
|
+
subject { Beso }
|
73
|
+
it { should respond_to( :secret_key ) }
|
74
|
+
its( :secret_key ){ should be_nil }
|
75
|
+
end
|
76
|
+
|
77
|
+
describe '#bucket_name' do
|
78
|
+
subject { Beso }
|
79
|
+
it { should respond_to( :bucket_name ) }
|
80
|
+
its( :bucket_name ){ should be_nil}
|
81
|
+
end
|
82
|
+
|
83
|
+
describe '#aws_region' do
|
84
|
+
subject { Beso }
|
85
|
+
it { should respond_to( :aws_region ) }
|
86
|
+
its( :aws_region ){ should be_nil }
|
87
|
+
end
|
51
88
|
end
|
data/spec/beso/job_spec.rb
CHANGED
@@ -2,17 +2,46 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe Beso::Job do
|
4
4
|
|
5
|
+
before do
|
6
|
+
User.destroy_all
|
7
|
+
end
|
8
|
+
|
5
9
|
describe 'to_csv' do
|
6
10
|
subject { Beso::Job.new :message_sent, :table => :users }
|
7
11
|
|
8
|
-
before do
|
9
|
-
User.destroy_all
|
10
|
-
end
|
11
|
-
|
12
12
|
let!( :foo ){ User.create! :name => 'Foo' }
|
13
13
|
let!( :bar ){ User.create! :name => 'Bar' }
|
14
14
|
|
15
|
-
context '
|
15
|
+
context 'without an identity defined' do
|
16
|
+
before do
|
17
|
+
subject.timestamp :created_at
|
18
|
+
end
|
19
|
+
it 'should raise an error' do
|
20
|
+
expect { subject.to_csv }.to raise_error( Beso::MissingIdentityError )
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'without a timestamp defined' do
|
25
|
+
before do
|
26
|
+
subject.identity { |user| user.id }
|
27
|
+
end
|
28
|
+
it 'should raise an error' do
|
29
|
+
expect { subject.to_csv }.to raise_error( Beso::MissingTimestampError )
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'with a timestamp that is not a symbol' do
|
34
|
+
it 'should raise an error' do
|
35
|
+
expect { subject.timestamp 123 }.to raise_error( Beso::InvalidTimestampError )
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'with only the mandatory columns defined' do
|
40
|
+
before do
|
41
|
+
subject.identity { |user| user.id }
|
42
|
+
subject.timestamp :created_at
|
43
|
+
end
|
44
|
+
|
16
45
|
its( :to_csv ){ should eq( <<-EOS
|
17
46
|
Identity,Timestamp,Event
|
18
47
|
#{foo.id},#{foo.created_at.to_i},Message Sent
|
@@ -21,61 +50,234 @@ Identity,Timestamp,Event
|
|
21
50
|
) }
|
22
51
|
end
|
23
52
|
|
24
|
-
context 'with
|
53
|
+
context 'with custom properties defined' do
|
25
54
|
before do
|
26
|
-
subject.
|
55
|
+
subject.identity { |user| user.id }
|
56
|
+
subject.timestamp :created_at
|
57
|
+
subject.prop( :user_name ){ |user| user.name }
|
27
58
|
end
|
28
59
|
|
29
60
|
its( :to_csv ){ should eq( <<-EOS
|
30
|
-
Identity,Timestamp,Event,Prop:Name
|
31
|
-
#{foo.id},#{foo.created_at.to_i},Message Sent
|
32
|
-
#{bar.id},#{bar.created_at.to_i},Message Sent
|
61
|
+
Identity,Timestamp,Event,Prop:User Name
|
62
|
+
#{foo.id},#{foo.created_at.to_i},Message Sent,Foo
|
63
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent,Bar
|
64
|
+
EOS
|
65
|
+
) }
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'with literal properties defined' do
|
69
|
+
before do
|
70
|
+
subject.identity 22
|
71
|
+
subject.timestamp :created_at
|
72
|
+
subject.prop :foo, 'bar'
|
73
|
+
end
|
74
|
+
|
75
|
+
its( :to_csv ){ should eq( <<-EOS
|
76
|
+
Identity,Timestamp,Event,Prop:Foo
|
77
|
+
22,#{foo.created_at.to_i},Message Sent,bar
|
78
|
+
22,#{bar.created_at.to_i},Message Sent,bar
|
33
79
|
EOS
|
34
80
|
) }
|
35
81
|
end
|
36
82
|
|
37
|
-
context 'with
|
83
|
+
context 'with symbol properties defined' do
|
38
84
|
before do
|
39
|
-
subject.
|
85
|
+
subject.identity :id
|
86
|
+
subject.timestamp :created_at
|
87
|
+
subject.prop :name
|
40
88
|
end
|
41
89
|
|
42
90
|
its( :to_csv ){ should eq( <<-EOS
|
43
|
-
Identity,Timestamp,Event,Prop:
|
91
|
+
Identity,Timestamp,Event,Prop:Name
|
44
92
|
#{foo.id},#{foo.created_at.to_i},Message Sent,#{foo.name}
|
45
93
|
#{bar.id},#{bar.created_at.to_i},Message Sent,#{bar.name}
|
46
94
|
EOS
|
47
95
|
) }
|
48
96
|
end
|
49
97
|
|
50
|
-
context 'with
|
98
|
+
context 'with 10 custom properties defined' do
|
51
99
|
before do
|
52
|
-
subject.
|
53
|
-
|
100
|
+
subject.identity 22
|
101
|
+
subject.timestamp :created_at
|
102
|
+
(1..10).each do |i|
|
103
|
+
subject.prop :"foo #{i}", i
|
54
104
|
end
|
55
105
|
end
|
56
106
|
|
57
107
|
its( :to_csv ){ should eq( <<-EOS
|
58
|
-
Identity,Timestamp,Event,Prop:
|
59
|
-
|
60
|
-
|
108
|
+
Identity,Timestamp,Event,Prop:Foo 1,Prop:Foo 2,Prop:Foo 3,Prop:Foo 4,Prop:Foo 5,Prop:Foo 6,Prop:Foo 7,Prop:Foo 8,Prop:Foo 9,Prop:Foo 10
|
109
|
+
22,#{foo.created_at.to_i},Message Sent,1,2,3,4,5,6,7,8,9,10
|
110
|
+
22,#{bar.created_at.to_i},Message Sent,1,2,3,4,5,6,7,8,9,10
|
61
111
|
EOS
|
62
|
-
)}
|
112
|
+
) }
|
63
113
|
end
|
64
114
|
|
65
|
-
context 'with
|
115
|
+
context 'with more than 10 custom properties defined' do
|
66
116
|
before do
|
67
|
-
subject.
|
68
|
-
|
69
|
-
|
117
|
+
subject.identity 22
|
118
|
+
subject.timestamp :created_at
|
119
|
+
end
|
120
|
+
it 'should raise an error' do
|
121
|
+
expect {
|
122
|
+
(1..11).each do |i|
|
123
|
+
subject.prop :"foo #{i}", i
|
124
|
+
end
|
125
|
+
}.to raise_error Beso::TooManyPropertiesError
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
context 'with custom options given to #to_csv' do
|
130
|
+
before do
|
131
|
+
subject.identity { |user| user.id }
|
132
|
+
subject.timestamp :created_at
|
133
|
+
end
|
134
|
+
|
135
|
+
it 'should support all options that CSV supports' do
|
136
|
+
subject.to_csv( :force_quotes => true, :col_sep => ';' ).should eq( <<-EOS
|
137
|
+
"Identity";"Timestamp";"Event"
|
138
|
+
"#{foo.id}";"#{foo.created_at.to_i}";"Message Sent"
|
139
|
+
"#{bar.id}";"#{bar.created_at.to_i}";"Message Sent"
|
140
|
+
EOS
|
141
|
+
)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
context 'with custom options given to constructor' do
|
146
|
+
subject { Beso::Job.new :message_sent, :table => :users, :col_sep => ';' }
|
147
|
+
|
148
|
+
before do
|
149
|
+
subject.identity { |user| user.id }
|
150
|
+
subject.timestamp :created_at
|
70
151
|
end
|
71
152
|
|
72
153
|
its( :to_csv ){ should eq( <<-EOS
|
73
|
-
Identity
|
74
|
-
#{foo.id}
|
75
|
-
#{bar.id}
|
154
|
+
Identity;Timestamp;Event
|
155
|
+
#{foo.id};#{foo.created_at.to_i};Message Sent
|
156
|
+
#{bar.id};#{bar.created_at.to_i};Message Sent
|
76
157
|
EOS
|
77
|
-
)}
|
158
|
+
) }
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'when no records match the query' do
|
162
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
163
|
+
|
164
|
+
before do
|
165
|
+
User.destroy_all
|
166
|
+
subject.identity { |user| user.id }
|
167
|
+
subject.timestamp :created_at
|
168
|
+
end
|
169
|
+
|
170
|
+
its( :to_csv ){ should be_nil }
|
78
171
|
end
|
79
172
|
end
|
80
173
|
|
174
|
+
describe 'with since specified' do
|
175
|
+
let!( :foo ){ User.create :name => 'Foo', :created_at => 100, :updated_at => 300 }
|
176
|
+
let!( :bar ){ User.create :name => 'Bar', :created_at => 200, :updated_at => 200 }
|
177
|
+
let!( :baz ){ User.create :name => 'Baz', :created_at => 300, :updated_at => 300 }
|
178
|
+
|
179
|
+
context 'in the constructor' do
|
180
|
+
context 'and the timestamp keyed on `created_at`' do
|
181
|
+
subject { Beso::Job.new :message_sent, :table => :users, :since => 101 }
|
182
|
+
|
183
|
+
before do
|
184
|
+
subject.identity { |user| user.id }
|
185
|
+
subject.timestamp :created_at
|
186
|
+
end
|
187
|
+
|
188
|
+
its( :to_csv ){ should eq( <<-EOS
|
189
|
+
Identity,Timestamp,Event
|
190
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent
|
191
|
+
#{baz.id},#{baz.created_at.to_i},Message Sent
|
192
|
+
EOS
|
193
|
+
) }
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'in to_csv' do
|
198
|
+
context 'and the timestamp keyed on `created_at`' do
|
199
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
200
|
+
|
201
|
+
before do
|
202
|
+
subject.identity { |user| user.id }
|
203
|
+
subject.timestamp :created_at
|
204
|
+
end
|
205
|
+
|
206
|
+
it 'should find records after `since`' do
|
207
|
+
subject.to_csv( :since => 101 ).should eq( <<-EOS
|
208
|
+
Identity,Timestamp,Event
|
209
|
+
#{bar.id},#{bar.created_at.to_i},Message Sent
|
210
|
+
#{baz.id},#{baz.created_at.to_i},Message Sent
|
211
|
+
EOS
|
212
|
+
)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
context 'in the constructor' do
|
218
|
+
context 'and the timestamp keyed on `updated_at`' do
|
219
|
+
subject { Beso::Job.new :message_sent, :table => :users, :since => 201 }
|
220
|
+
|
221
|
+
before do
|
222
|
+
subject.identity { |user| user.id }
|
223
|
+
subject.timestamp :updated_at
|
224
|
+
end
|
225
|
+
|
226
|
+
its( :to_csv ){ should eq( <<-EOS
|
227
|
+
Identity,Timestamp,Event
|
228
|
+
#{foo.id},#{foo.updated_at.to_i},Message Sent
|
229
|
+
#{baz.id},#{baz.updated_at.to_i},Message Sent
|
230
|
+
EOS
|
231
|
+
) }
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
context 'in to_csv' do
|
236
|
+
context 'and the timestamp keyed on `updated_at`' do
|
237
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
238
|
+
|
239
|
+
before do
|
240
|
+
subject.identity { |user| user.id }
|
241
|
+
subject.timestamp :updated_at
|
242
|
+
end
|
243
|
+
|
244
|
+
it 'should find records after `since`' do
|
245
|
+
subject.to_csv( :since => 201 ).should eq( <<-EOS
|
246
|
+
Identity,Timestamp,Event
|
247
|
+
#{foo.id},#{foo.updated_at.to_i},Message Sent
|
248
|
+
#{baz.id},#{baz.updated_at.to_i},Message Sent
|
249
|
+
EOS
|
250
|
+
)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
describe '#last_timestamp' do
|
257
|
+
let!( :foo ){ User.create :name => 'Foo', :created_at => 100, :updated_at => 301 }
|
258
|
+
let!( :bar ){ User.create :name => 'Bar', :created_at => 200, :updated_at => 200 }
|
259
|
+
let!( :baz ){ User.create :name => 'Baz', :created_at => 300, :updated_at => 300 }
|
260
|
+
|
261
|
+
subject { Beso::Job.new :message_sent, :table => :users }
|
262
|
+
|
263
|
+
before do
|
264
|
+
subject.identity :id
|
265
|
+
end
|
266
|
+
|
267
|
+
context 'with the timestamp keyed on `created_at`' do
|
268
|
+
before do
|
269
|
+
subject.timestamp :created_at
|
270
|
+
end
|
271
|
+
|
272
|
+
its( :last_timestamp ){ should eq( 300 ) }
|
273
|
+
end
|
274
|
+
|
275
|
+
context 'with the timestamp keyed on `updated_at`' do
|
276
|
+
before do
|
277
|
+
subject.timestamp :updated_at
|
278
|
+
end
|
279
|
+
|
280
|
+
its( :last_timestamp ){ should eq( 301 ) }
|
281
|
+
end
|
282
|
+
end
|
81
283
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beso
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-04-
|
12
|
+
date: 2012-04-12 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -28,13 +28,13 @@ dependencies:
|
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 3.0.10
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: fastercsv
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
35
|
- - ! '>='
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version:
|
37
|
+
version: 1.5.4
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,23 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
45
|
+
version: 1.5.4
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: fog
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.3.1
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.3.1
|
46
62
|
- !ruby/object:Gem::Dependency
|
47
63
|
name: sqlite3
|
48
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,12 +123,15 @@ files:
|
|
107
123
|
- beso.gemspec
|
108
124
|
- gemfiles/rails-3.0.10.gemfile
|
109
125
|
- gemfiles/rails-3.0.10.gemfile.lock
|
110
|
-
- lib/beso.rake
|
111
126
|
- lib/beso.rb
|
112
127
|
- lib/beso/config.rb
|
128
|
+
- lib/beso/connection.rb
|
129
|
+
- lib/beso/csv.rb
|
113
130
|
- lib/beso/job.rb
|
114
131
|
- lib/beso/railtie.rb
|
115
132
|
- lib/beso/version.rb
|
133
|
+
- lib/tasks/beso.rake
|
134
|
+
- spec/beso/aws_spec.rb
|
116
135
|
- spec/beso/config_spec.rb
|
117
136
|
- spec/beso/job_spec.rb
|
118
137
|
- spec/rails/.gitkeep
|
@@ -143,6 +162,7 @@ signing_key:
|
|
143
162
|
specification_version: 3
|
144
163
|
summary: Sync your KISSmetrics history, guapo!
|
145
164
|
test_files:
|
165
|
+
- spec/beso/aws_spec.rb
|
146
166
|
- spec/beso/config_spec.rb
|
147
167
|
- spec/beso/job_spec.rb
|
148
168
|
- spec/rails/.gitkeep
|