td 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/ChangeLog +45 -14
- data/README.rdoc +50 -28
- data/dist/exe.rake +2 -2
- data/dist/pkg.rake +2 -2
- data/lib/td/command/bulk_import.rb +1 -5
- data/lib/td/command/common.rb +116 -20
- data/lib/td/command/help.rb +1 -1
- data/lib/td/command/import.rb +50 -5
- data/lib/td/command/job.rb +57 -15
- data/lib/td/command/list.rb +90 -87
- data/lib/td/command/runner.rb +33 -21
- data/lib/td/command/sched.rb +13 -5
- data/lib/td/command/server.rb +6 -3
- data/lib/td/command/table.rb +75 -61
- data/lib/td/updater.rb +31 -62
- data/lib/td/version.rb +1 -1
- data/spec/td/common_spec.rb +192 -0
- data/spec/td/updater_spec.rb +9 -3
- data/td.gemspec +2 -1
- metadata +7 -5
data/lib/td/command/table.rb
CHANGED
@@ -11,6 +11,7 @@ module Command
|
|
11
11
|
COLLECTION ITEMS KEYS LINES STORED SEQUENCEFILE TEXTFILE INPUTFORMAT OUTPUTFORMAT LOCATION TABLESAMPLE BUCKET OUT
|
12
12
|
OF CAST ADD REPLACE COLUMNS RLIKE REGEXP TEMPORARY FUNCTION EXPLAIN EXTENDED SERDE WITH SERDEPROPERTIES LIMIT SET TBLPROPERTIES
|
13
13
|
]
|
14
|
+
KEY_NUM_LIMIT = 512
|
14
15
|
|
15
16
|
def table_create(op)
|
16
17
|
type = nil
|
@@ -485,7 +486,10 @@ module Command
|
|
485
486
|
client.create_database(db_name)
|
486
487
|
$stderr.puts "Database '#{db_name}' is created."
|
487
488
|
rescue AlreadyExistsError
|
489
|
+
# do nothing
|
488
490
|
end
|
491
|
+
rescue ForbiddenError
|
492
|
+
# do nothing
|
489
493
|
end
|
490
494
|
|
491
495
|
API.validate_table_name(table_name)
|
@@ -510,7 +514,7 @@ module Command
|
|
510
514
|
parser = MessagePackParser.new(time_key)
|
511
515
|
end
|
512
516
|
|
513
|
-
else
|
517
|
+
else # apache, syslog
|
514
518
|
regexp, names, time_format = IMPORT_TEMPLATES[format]
|
515
519
|
if !regexp || !names || !time_format
|
516
520
|
$stderr.puts "Unknown format '#{format}'"
|
@@ -519,26 +523,40 @@ module Command
|
|
519
523
|
parser = TextParser.new(names, regexp, time_format)
|
520
524
|
end
|
521
525
|
|
522
|
-
|
526
|
+
begin
|
527
|
+
db = client.database(db_name)
|
528
|
+
rescue ForbiddenError => e
|
529
|
+
puts "Warning: database and table validation skipped - #{e.message}"
|
530
|
+
else
|
531
|
+
begin
|
532
|
+
table = db.table(table_name)
|
533
|
+
rescue ForbiddenError => e
|
534
|
+
puts "Warning: table validation skipped - #{e.message}"
|
535
|
+
end
|
536
|
+
end
|
523
537
|
|
524
538
|
require 'zlib'
|
525
539
|
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
540
|
+
begin
|
541
|
+
files = paths.map {|path|
|
542
|
+
if path == '-'
|
543
|
+
$stdin
|
544
|
+
elsif path =~ /\.gz$/
|
545
|
+
require 'td/compat_gzip_reader'
|
546
|
+
Zlib::GzipReader.open(path)
|
547
|
+
else
|
548
|
+
File.open(path)
|
549
|
+
end
|
550
|
+
}
|
551
|
+
rescue Errno::ENOENT => e
|
552
|
+
raise ImportError, e.message
|
553
|
+
end
|
536
554
|
|
537
555
|
require 'msgpack'
|
538
556
|
require 'tempfile'
|
539
557
|
#require 'thread'
|
540
558
|
|
541
|
-
files.zip(paths).each {|file,path|
|
559
|
+
files.zip(paths).each {|file, path|
|
542
560
|
import_log_file(file, path, client, db_name, table_name, parser)
|
543
561
|
}
|
544
562
|
|
@@ -568,10 +586,11 @@ module Command
|
|
568
586
|
|
569
587
|
n += 1
|
570
588
|
x += 1
|
571
|
-
if n % 10000 == 0
|
589
|
+
if n % 10000 == 0 # by records imported
|
572
590
|
puts " imported #{n} entries from #{path}..."
|
573
591
|
|
574
|
-
|
592
|
+
# TODO size
|
593
|
+
elsif out.pos > 1024 * 1024 # by 1 MB chunks
|
575
594
|
puts " imported #{n} entries from #{path}..."
|
576
595
|
begin
|
577
596
|
writer.finish
|
@@ -592,6 +611,7 @@ module Command
|
|
592
611
|
end
|
593
612
|
}
|
594
613
|
|
614
|
+
# if there is anything parse but not imported yet
|
595
615
|
if x != 0
|
596
616
|
writer.finish
|
597
617
|
size = out.pos
|
@@ -602,6 +622,11 @@ module Command
|
|
602
622
|
client.import(db_name, table_name, "msgpack.gz", out, size)
|
603
623
|
end
|
604
624
|
|
625
|
+
# throw an exception if no record is imported
|
626
|
+
if n == 0
|
627
|
+
raise ImportError, "no valid record to import from #{path}"
|
628
|
+
end
|
629
|
+
|
605
630
|
puts " imported #{n} entries from #{path}."
|
606
631
|
$stderr.puts normalized_message if has_bignum
|
607
632
|
ensure
|
@@ -661,75 +686,64 @@ module Command
|
|
661
686
|
end
|
662
687
|
end
|
663
688
|
|
664
|
-
class
|
689
|
+
# Generic class for both JSON and MessagePack parsers to
|
690
|
+
# reduce code duplication
|
691
|
+
class StructuredParser
|
692
|
+
def sanitize_record(record, &block)
|
693
|
+
unless record.is_a?(Hash)
|
694
|
+
raise "record must be a Hash"
|
695
|
+
end
|
696
|
+
|
697
|
+
time = record[@time_key]
|
698
|
+
unless time
|
699
|
+
raise "record doesn't have '#{@time_key}' column"
|
700
|
+
end
|
701
|
+
|
702
|
+
if record.size > KEY_NUM_LIMIT
|
703
|
+
raise "record contains too many keys (#{record.size}, max allowed #{KEY_NUM_LIMIT})"
|
704
|
+
end
|
705
|
+
|
706
|
+
case time
|
707
|
+
when Integer
|
708
|
+
# do nothing
|
709
|
+
else
|
710
|
+
time = Time.parse(time.to_s).to_i
|
711
|
+
end
|
712
|
+
record['time'] = time
|
713
|
+
|
714
|
+
block.call(record)
|
715
|
+
end
|
716
|
+
protected :sanitize_record
|
717
|
+
end
|
718
|
+
|
719
|
+
class JsonParser < StructuredParser
|
665
720
|
def initialize(time_key)
|
666
721
|
require 'json'
|
667
722
|
@time_key = time_key
|
668
723
|
end
|
669
724
|
|
670
725
|
def call(file, path, &block)
|
671
|
-
i = 0
|
672
726
|
file.each_line {|line|
|
673
|
-
i += 1
|
674
727
|
begin
|
675
728
|
record = JSON.parse(line)
|
676
|
-
|
677
|
-
unless record.is_a?(Hash)
|
678
|
-
raise "record must be a Hash"
|
679
|
-
end
|
680
|
-
|
681
|
-
time = record[@time_key]
|
682
|
-
unless time
|
683
|
-
raise "record doesn't have '#{@time_key}' column"
|
684
|
-
end
|
685
|
-
|
686
|
-
case time
|
687
|
-
when Integer
|
688
|
-
# do nothing
|
689
|
-
else
|
690
|
-
time = Time.parse(time.to_s).to_i
|
691
|
-
end
|
692
|
-
record['time'] = time
|
693
|
-
|
694
|
-
block.call(record)
|
695
|
-
|
729
|
+
sanitize_record(record, &block)
|
696
730
|
rescue
|
697
|
-
$stderr.puts " skipped: #{$!}: #{
|
731
|
+
$stderr.puts " skipped: #{$!}: #{record.to_json}"
|
698
732
|
end
|
699
733
|
}
|
700
734
|
end
|
701
735
|
end
|
702
736
|
|
703
|
-
class MessagePackParser
|
737
|
+
class MessagePackParser < StructuredParser
|
704
738
|
def initialize(time_key)
|
705
739
|
require 'msgpack'
|
706
740
|
@time_key = time_key
|
707
741
|
end
|
708
742
|
|
709
743
|
def call(file, path, &block)
|
710
|
-
i = 0
|
711
744
|
MessagePack::Unpacker.new(file).each {|record|
|
712
|
-
i += 1
|
713
745
|
begin
|
714
|
-
|
715
|
-
raise "record must be a Hash"
|
716
|
-
end
|
717
|
-
|
718
|
-
time = record[@time_key]
|
719
|
-
unless time
|
720
|
-
raise "record doesn't have '#{@time_key}' column"
|
721
|
-
end
|
722
|
-
|
723
|
-
case time
|
724
|
-
when Integer
|
725
|
-
# do nothing
|
726
|
-
else
|
727
|
-
time = Time.parse(time.to_s).to_i
|
728
|
-
end
|
729
|
-
record['time'] = time
|
730
|
-
|
731
|
-
block.call(record)
|
732
|
-
|
746
|
+
sanitize_record(record, &block)
|
733
747
|
rescue
|
734
748
|
$stderr.puts " skipped: #{$!}: #{record.to_json}"
|
735
749
|
end
|
data/lib/td/updater.rb
CHANGED
@@ -103,7 +103,7 @@ module Updater
|
|
103
103
|
when on_mac?
|
104
104
|
'pkg'
|
105
105
|
else
|
106
|
-
raise_error "
|
106
|
+
raise_error "Environment not supported"
|
107
107
|
end
|
108
108
|
end
|
109
109
|
|
@@ -132,7 +132,8 @@ module Updater
|
|
132
132
|
when Net::HTTPSuccess then response.body
|
133
133
|
when Net::HTTPRedirection then fetch(response['Location'])
|
134
134
|
else
|
135
|
-
raise
|
135
|
+
raise Command::UpdateError,
|
136
|
+
"An error occurred when fetching from '#{url}'."
|
136
137
|
response.error!
|
137
138
|
end
|
138
139
|
end
|
@@ -141,6 +142,10 @@ module Updater
|
|
141
142
|
ENV['TD_TOOLBELT_UPDATE_ROOT'] || "http://toolbelt.treasuredata.com"
|
142
143
|
end
|
143
144
|
|
145
|
+
def maven_repo_root
|
146
|
+
ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "http://central.maven.org"
|
147
|
+
end
|
148
|
+
|
144
149
|
def self.version_endpoint
|
145
150
|
"#{endpoint_root}/version.#{package_category}"
|
146
151
|
end
|
@@ -252,8 +257,8 @@ module Updater
|
|
252
257
|
#
|
253
258
|
|
254
259
|
# locate the root of the td package which is 3 folders up from the location of this file
|
255
|
-
def jarfile_dest_path
|
256
|
-
File.join(
|
260
|
+
def self.jarfile_dest_path
|
261
|
+
File.join(home_directory, ".td", "java")
|
257
262
|
end
|
258
263
|
|
259
264
|
private
|
@@ -280,10 +285,13 @@ module Updater
|
|
280
285
|
return true
|
281
286
|
elsif response.class == Net::HTTPFound || \
|
282
287
|
response.class == Net::HTTPRedirection
|
283
|
-
|
288
|
+
unless ENV['TD_TOOLBELT_DEBUG'].nil?
|
289
|
+
puts "redirect '#{url}' to '#{response['Location']}'... "
|
290
|
+
end
|
284
291
|
return stream_fetch(response['Location'], binfile, &progress)
|
285
292
|
else
|
286
|
-
raise
|
293
|
+
raise Command::UpdateError,
|
294
|
+
"An error occurred when fetching from '#{uri}' " +
|
287
295
|
"(#{response.class.to_s}: #{response.message})."
|
288
296
|
return false
|
289
297
|
end
|
@@ -296,23 +304,25 @@ module Updater
|
|
296
304
|
require 'open-uri'
|
297
305
|
require 'fileutils'
|
298
306
|
|
299
|
-
maven_repo = "
|
307
|
+
maven_repo = "#{maven_repo_root}/maven2/com/treasuredata/td-import"
|
300
308
|
|
301
309
|
begin
|
302
310
|
xml = Updater.fetch("#{maven_repo}/maven-metadata.xml")
|
303
311
|
rescue Exception => exc
|
304
312
|
raise Command::UpdateError,
|
305
|
-
"There was a problem accessing the remote XML resource
|
306
|
-
"(#{exc.class.to_s}: #{exc.message})"
|
313
|
+
"There was a problem accessing the remote XML resource " +
|
314
|
+
"'#{maven_repo}/maven-metadata.xml' (#{exc.class.to_s}: #{exc.message})"
|
307
315
|
end
|
308
316
|
if xml.nil? || xml.empty?
|
309
317
|
raise Command::UpdateError,
|
310
|
-
"The remote XML resource '#{maven_repo}/maven-metadata.xml'
|
318
|
+
"The remote XML resource '#{maven_repo}/maven-metadata.xml' " +
|
319
|
+
"returned an empty file."
|
311
320
|
end
|
312
321
|
|
313
322
|
# read version and update date from the xml file
|
314
323
|
doc = REXML::Document.new(xml)
|
315
|
-
updated = Time.strptime(REXML::XPath.match(doc,
|
324
|
+
updated = Time.strptime(REXML::XPath.match(doc,
|
325
|
+
'/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
|
316
326
|
version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
|
317
327
|
|
318
328
|
# Convert into UTF to compare time correctly
|
@@ -320,8 +330,8 @@ module Updater
|
|
320
330
|
last_updated = existent_jar_updated_time
|
321
331
|
|
322
332
|
if updated > last_updated
|
323
|
-
FileUtils.mkdir_p(jarfile_dest_path) unless File.exists?(jarfile_dest_path)
|
324
|
-
Dir.chdir jarfile_dest_path
|
333
|
+
FileUtils.mkdir_p(Updater.jarfile_dest_path) unless File.exists?(Updater.jarfile_dest_path)
|
334
|
+
Dir.chdir Updater.jarfile_dest_path
|
325
335
|
|
326
336
|
File.open('VERSION', 'w') {|f|
|
327
337
|
if hourly
|
@@ -334,17 +344,18 @@ module Updater
|
|
334
344
|
f.print "#{version} #{updated}"
|
335
345
|
}
|
336
346
|
|
347
|
+
status = nil
|
337
348
|
indicator = Command::TimeBasedDownloadProgressIndicator.new(
|
338
349
|
"Updating td-import.jar", Time.new.to_i, 2)
|
339
|
-
|
340
|
-
|
341
|
-
|
350
|
+
File.open('td-import.jar.new', 'wb') {|binfile|
|
351
|
+
status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
|
352
|
+
indicator.update
|
353
|
+
}
|
342
354
|
}
|
343
|
-
binfile.close
|
344
355
|
indicator.finish()
|
345
356
|
|
346
357
|
if status
|
347
|
-
puts "Installed td-import.jar v#{version} in '#{jarfile_dest_path}'.\n"
|
358
|
+
puts "Installed td-import.jar v#{version} in '#{Updater.jarfile_dest_path}'.\n"
|
348
359
|
File.rename 'td-import.jar.new', 'td-import.jar'
|
349
360
|
else
|
350
361
|
puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
|
@@ -379,12 +390,12 @@ module Updater
|
|
379
390
|
|
380
391
|
private
|
381
392
|
def last_jar_autoupdate_timestamp
|
382
|
-
File.join(jarfile_dest_path, "td-import-java.version")
|
393
|
+
File.join(Updater.jarfile_dest_path, "td-import-java.version")
|
383
394
|
end
|
384
395
|
|
385
396
|
private
|
386
397
|
def existent_jar_updated_time
|
387
|
-
files = find_files("td-import-java.version", [jarfile_dest_path])
|
398
|
+
files = Command.find_files("td-import-java.version", [Updater.jarfile_dest_path])
|
388
399
|
if files.empty?
|
389
400
|
return Time.at(0)
|
390
401
|
end
|
@@ -399,47 +410,5 @@ module Updater
|
|
399
410
|
time
|
400
411
|
end
|
401
412
|
|
402
|
-
#
|
403
|
-
# Helpers
|
404
|
-
#
|
405
|
-
def find_files(glob, locations)
|
406
|
-
files = []
|
407
|
-
locations.each {|loc|
|
408
|
-
files = Dir.glob("#{loc}/#{glob}")
|
409
|
-
break unless files.empty?
|
410
|
-
}
|
411
|
-
files
|
412
|
-
end
|
413
|
-
|
414
|
-
def find_version_file
|
415
|
-
version = find_files('VERSION', [jarfile_dest_path])
|
416
|
-
if version.empty?
|
417
|
-
$stderr.puts "Cannot find VERSION file in '#{jarfile_dest_path}'."
|
418
|
-
exit 10
|
419
|
-
end
|
420
|
-
version.first
|
421
|
-
end
|
422
|
-
|
423
|
-
def find_td_import_jar
|
424
|
-
jar = find_files('td-import.jar', [jarfile_dest_path])
|
425
|
-
if jar.empty?
|
426
|
-
$stderr.puts "Cannot find td-import.jar in '#{jarfile_dest_path}'."
|
427
|
-
exit 10
|
428
|
-
end
|
429
|
-
jar.first
|
430
|
-
end
|
431
|
-
|
432
|
-
def find_logging_property
|
433
|
-
installed_path = File.join(File.expand_path('../..', File.dirname(__FILE__)), 'java')
|
434
|
-
|
435
|
-
config = find_files("logging.properties", [installed_path])
|
436
|
-
if config.empty?
|
437
|
-
puts "Cannot find 'logging.properties' file in '#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
|
438
|
-
[]
|
439
|
-
else
|
440
|
-
config.first
|
441
|
-
end
|
442
|
-
end
|
443
|
-
|
444
413
|
end # module Updater
|
445
414
|
end # module TreasureData
|
data/lib/td/version.rb
CHANGED
@@ -0,0 +1,192 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'td/command/common'
|
3
|
+
|
4
|
+
module TreasureData::Command
|
5
|
+
describe 'humanize_bytesize' do
|
6
|
+
describe 'for values < 1024' do
|
7
|
+
values = [0, 1, 10, 1023]
|
8
|
+
values.each {|v|
|
9
|
+
it "uses B as label and has no suffix (#{v})" do
|
10
|
+
TreasureData::Command::humanize_bytesize(v, 1).should == "#{v} B"
|
11
|
+
end
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
describe 'for 1024' do
|
16
|
+
it 'uses kB and does not have a suffix' do
|
17
|
+
TreasureData::Command::humanize_bytesize(1024).should == "1 kB"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
describe 'for values between 1025 and (1024^2 - 1)' do
|
21
|
+
base = 1024
|
22
|
+
values = [
|
23
|
+
[base + 1, "1.0"],
|
24
|
+
[base + 2, "1.0"],
|
25
|
+
[base * 1024 - 1, "1023.9"]
|
26
|
+
]
|
27
|
+
values.each {|val, exp|
|
28
|
+
it "uses kB as label and has a suffix (#{val})" do
|
29
|
+
result = TreasureData::Command::humanize_bytesize(val, 1)
|
30
|
+
expect(result).to eq("#{exp} kB")
|
31
|
+
end
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
35
|
+
describe 'for 1024^2' do
|
36
|
+
it 'uses MB and does not have a suffix' do
|
37
|
+
TreasureData::Command::humanize_bytesize(1024 ** 2).should == "1 MB"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
describe 'for values between (1024^2 + 1) and (1024^3 - 1)' do
|
41
|
+
base = 1024 ** 2
|
42
|
+
values = [
|
43
|
+
[base + 1, "1.0"],
|
44
|
+
[base + 2, "1.0"],
|
45
|
+
[base * 1024 - 1, "1023.9"]
|
46
|
+
]
|
47
|
+
values.each {|val, exp|
|
48
|
+
it "uses MB as label and has a suffix (#{val})" do
|
49
|
+
result = TreasureData::Command::humanize_bytesize(val, 1)
|
50
|
+
expect(result).to eq("#{exp} MB")
|
51
|
+
end
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
describe 'for 1024^3' do
|
56
|
+
it 'uses GB and does not have a suffix' do
|
57
|
+
TreasureData::Command::humanize_bytesize(1024 ** 3).should == "1 GB"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
describe 'for values between (1024^3 + 1) and (1024^4 - 1)' do
|
61
|
+
base = 1024 ** 3
|
62
|
+
values = [
|
63
|
+
[base + 1, "1.0"],
|
64
|
+
[base + 2, "1.0"],
|
65
|
+
[base * 1024 - 1, "1023.9"]
|
66
|
+
]
|
67
|
+
values.each {|val, exp|
|
68
|
+
it "uses GB as label and has a suffix (#{val})" do
|
69
|
+
result = TreasureData::Command::humanize_bytesize(val, 1)
|
70
|
+
expect(result).to eq("#{exp} GB")
|
71
|
+
end
|
72
|
+
}
|
73
|
+
end
|
74
|
+
|
75
|
+
describe 'for 1024^4' do
|
76
|
+
it 'uses TB and does not have a suffix' do
|
77
|
+
TreasureData::Command::humanize_bytesize(1024 ** 4).should == "1 TB"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
describe 'for values between (1024^4 + 1) and (1024^5 - 1)' do
|
81
|
+
base = 1024 ** 4
|
82
|
+
values = [
|
83
|
+
[base + 1, "1.0"],
|
84
|
+
[base + 2, "1.0"],
|
85
|
+
[base * 1024 - 1, "1023.9"]
|
86
|
+
]
|
87
|
+
values.each {|val, exp|
|
88
|
+
it "uses TB as label and has a suffix (#{val})" do
|
89
|
+
result = TreasureData::Command::humanize_bytesize(val, 1)
|
90
|
+
expect(result).to eq("#{exp} TB")
|
91
|
+
end
|
92
|
+
}
|
93
|
+
end
|
94
|
+
|
95
|
+
describe 'for 1024^5' do
|
96
|
+
it 'uses TB and does not have a suffix' do
|
97
|
+
TreasureData::Command::humanize_bytesize(1024 ** 5).should == "1024 TB"
|
98
|
+
end
|
99
|
+
end
|
100
|
+
describe 'for values between (1024^5 + 1) and (1024^6 - 1)' do
|
101
|
+
base = 1024 ** 5
|
102
|
+
values = [
|
103
|
+
[base + 1, "1024.0"],
|
104
|
+
[base + 2, "1024.0"],
|
105
|
+
[base * 1024 - 1, "1048575.9"]
|
106
|
+
]
|
107
|
+
values.each {|val, exp|
|
108
|
+
it "uses TB as label and has a suffix (#{val})" do
|
109
|
+
result = TreasureData::Command::humanize_bytesize(val, 1)
|
110
|
+
expect(result).to eq("#{exp} TB")
|
111
|
+
end
|
112
|
+
}
|
113
|
+
end
|
114
|
+
|
115
|
+
describe 'shows 1 digit' do
|
116
|
+
it 'without second function argument' do
|
117
|
+
values = [1024 + 1024 / 2, "1.5"]
|
118
|
+
val, exp = values
|
119
|
+
result = TreasureData::Command::humanize_bytesize(val)
|
120
|
+
expect(result).to eq("#{exp} kB")
|
121
|
+
end
|
122
|
+
end
|
123
|
+
describe 'shows the correct number of digits specified by the second argument' do
|
124
|
+
(0...5).each {|i|
|
125
|
+
it "when = #{i}" do
|
126
|
+
val = 1024 + 1024 / 2
|
127
|
+
if i == 0
|
128
|
+
exp = 1.to_s
|
129
|
+
else
|
130
|
+
exp = sprintf "%.*f", i, 1.5
|
131
|
+
end
|
132
|
+
result = TreasureData::Command::humanize_bytesize(val, i)
|
133
|
+
expect(result).to eq("#{exp} kB")
|
134
|
+
end
|
135
|
+
}
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
describe 'SizeBasedDownloadProgressIndicator' do
|
140
|
+
it "shows in 1% increments with default 'perc_step'" do
|
141
|
+
size = 200
|
142
|
+
indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", size)
|
143
|
+
size_increments = 2
|
144
|
+
curr_size = 0
|
145
|
+
while (curr_size += size_increments) < size do
|
146
|
+
indicator.update(size_increments)
|
147
|
+
sleep(0.05)
|
148
|
+
end
|
149
|
+
indicator.finish
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
describe 'TimeBasedDownloadProgressIndicator' do
|
154
|
+
it "increments about every 2 seconds with default 'periodicity'" do
|
155
|
+
start_time = Time.now.to_i
|
156
|
+
indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time)
|
157
|
+
end_time = start_time + 10
|
158
|
+
last_time = start_time
|
159
|
+
while (curr_time = Time.now.to_i) < end_time do
|
160
|
+
ret = indicator.update
|
161
|
+
if ret == true
|
162
|
+
diff = curr_time - last_time
|
163
|
+
diff.should be >= 2
|
164
|
+
diff.should be < 3
|
165
|
+
last_time = curr_time
|
166
|
+
end
|
167
|
+
sleep(0.5)
|
168
|
+
end
|
169
|
+
indicator.finish
|
170
|
+
end
|
171
|
+
|
172
|
+
periodicities = [1, 2, 5]
|
173
|
+
periodicities.each {|periodicity|
|
174
|
+
it "increments about every #{periodicity} seconds with 'periodicity' = #{periodicity}" do
|
175
|
+
start_time = Time.now.to_i
|
176
|
+
indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time, periodicity)
|
177
|
+
end_time = start_time + 10
|
178
|
+
last_time = start_time
|
179
|
+
while (curr_time = Time.now.to_i) < end_time do
|
180
|
+
ret = indicator.update
|
181
|
+
if ret == true
|
182
|
+
(curr_time - last_time).should be >= periodicity
|
183
|
+
(curr_time - last_time).should be < (periodicity + 1)
|
184
|
+
last_time = curr_time
|
185
|
+
end
|
186
|
+
sleep(0.5)
|
187
|
+
end
|
188
|
+
indicator.finish
|
189
|
+
end
|
190
|
+
}
|
191
|
+
end
|
192
|
+
end
|
data/spec/td/updater_spec.rb
CHANGED
@@ -2,15 +2,21 @@ require 'spec_helper'
|
|
2
2
|
require 'td/updater'
|
3
3
|
|
4
4
|
module TreasureData::Updater
|
5
|
+
|
5
6
|
describe 'without the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
|
7
|
+
let :default_toolbelt_url do
|
8
|
+
"http://toolbelt.treasuredata.com"
|
9
|
+
end
|
10
|
+
|
6
11
|
describe 'endpoints methods' do
|
7
12
|
it 'use the default root path' do
|
8
|
-
TreasureData::Updater.endpoint_root.should ==
|
9
|
-
TreasureData::Updater.version_endpoint.should =~ Regexp.new(
|
10
|
-
TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(
|
13
|
+
TreasureData::Updater.endpoint_root.should == default_toolbelt_url
|
14
|
+
TreasureData::Updater.version_endpoint.should =~ Regexp.new(default_toolbelt_url)
|
15
|
+
TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(default_toolbelt_url)
|
11
16
|
end
|
12
17
|
end
|
13
18
|
end
|
19
|
+
|
14
20
|
describe 'with the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
|
15
21
|
before do
|
16
22
|
ENV['TD_TOOLBELT_UPDATE_ROOT'] = 'https://0.0.0.0:5000/'
|
data/td.gemspec
CHANGED
@@ -15,12 +15,13 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
16
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
17
|
gem.require_paths = ['lib']
|
18
|
+
gem.required_ruby_version = '>= 1.9'
|
18
19
|
|
19
20
|
gem.add_dependency "msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"]
|
20
21
|
gem.add_dependency "yajl-ruby", "~> 1.1"
|
21
22
|
gem.add_dependency "hirb", ">= 0.4.5"
|
22
23
|
gem.add_dependency "parallel", "~> 0.6.1"
|
23
|
-
gem.add_dependency "td-client", "~> 0.8.
|
24
|
+
gem.add_dependency "td-client", "~> 0.8.63"
|
24
25
|
gem.add_dependency "td-logger", "~> 0.3.21"
|
25
26
|
gem.add_dependency "rubyzip", "~> 0.9.9"
|
26
27
|
gem.add_development_dependency "rake", "~> 0.9"
|