imw 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. data/.gitignore +4 -1
  2. data/Rakefile +10 -0
  3. data/TODO +18 -0
  4. data/VERSION +1 -1
  5. data/bin/imw +1 -1
  6. data/etc/imwrc.rb +0 -50
  7. data/examples/dataset.rb +12 -0
  8. data/lib/imw/boot.rb +55 -9
  9. data/lib/imw/dataset/paths.rb +15 -24
  10. data/lib/imw/dataset/workflow.rb +131 -72
  11. data/lib/imw/dataset.rb +94 -186
  12. data/lib/imw/parsers/html_parser.rb +1 -1
  13. data/lib/imw/parsers.rb +1 -1
  14. data/lib/imw/repository.rb +3 -27
  15. data/lib/imw/resource.rb +190 -0
  16. data/lib/imw/resources/archive.rb +97 -0
  17. data/lib/imw/resources/archives_and_compressed/bz2.rb +18 -0
  18. data/lib/imw/resources/archives_and_compressed/gz.rb +18 -0
  19. data/lib/imw/resources/archives_and_compressed/rar.rb +23 -0
  20. data/lib/imw/resources/archives_and_compressed/tar.rb +23 -0
  21. data/lib/imw/resources/archives_and_compressed/tarbz2.rb +78 -0
  22. data/lib/imw/resources/archives_and_compressed/targz.rb +78 -0
  23. data/lib/imw/resources/archives_and_compressed/zip.rb +57 -0
  24. data/lib/imw/resources/archives_and_compressed.rb +32 -0
  25. data/lib/imw/resources/compressed_file.rb +89 -0
  26. data/lib/imw/resources/compressible.rb +77 -0
  27. data/lib/imw/resources/formats/delimited.rb +92 -0
  28. data/lib/imw/resources/formats/excel.rb +125 -0
  29. data/lib/imw/resources/formats/json.rb +53 -0
  30. data/lib/imw/resources/formats/sgml.rb +72 -0
  31. data/lib/imw/resources/formats/yaml.rb +53 -0
  32. data/lib/imw/resources/formats.rb +32 -0
  33. data/lib/imw/resources/local.rb +198 -0
  34. data/lib/imw/resources/remote.rb +110 -0
  35. data/lib/imw/resources/schemes/hdfs.rb +242 -0
  36. data/lib/imw/resources/schemes/http.rb +161 -0
  37. data/lib/imw/resources/schemes/s3.rb +137 -0
  38. data/lib/imw/resources/schemes.rb +19 -0
  39. data/lib/imw/resources.rb +118 -0
  40. data/lib/imw/runner.rb +5 -4
  41. data/lib/imw/transforms/archiver.rb +215 -0
  42. data/lib/imw/transforms/transferer.rb +103 -0
  43. data/lib/imw/transforms.rb +8 -0
  44. data/lib/imw/utils/error.rb +26 -30
  45. data/lib/imw/utils/extensions/array.rb +5 -15
  46. data/lib/imw/utils/extensions/hash.rb +6 -16
  47. data/lib/imw/utils/extensions/hpricot.rb +0 -14
  48. data/lib/imw/utils/extensions/string.rb +5 -15
  49. data/lib/imw/utils/extensions/symbol.rb +0 -13
  50. data/lib/imw/utils/extensions.rb +65 -0
  51. data/lib/imw/utils/log.rb +14 -13
  52. data/lib/imw/utils/misc.rb +0 -6
  53. data/lib/imw/utils/paths.rb +101 -42
  54. data/lib/imw/utils/version.rb +8 -9
  55. data/lib/imw/utils.rb +2 -18
  56. data/lib/imw.rb +92 -17
  57. data/spec/data/sample.csv +1 -1
  58. data/spec/data/sample.json +1 -0
  59. data/spec/data/sample.tsv +1 -1
  60. data/spec/data/sample.txt +1 -1
  61. data/spec/data/sample.xml +1 -1
  62. data/spec/data/sample.yaml +1 -1
  63. data/spec/imw/dataset/paths_spec.rb +32 -0
  64. data/spec/imw/dataset/workflow_spec.rb +41 -0
  65. data/spec/imw/resource_spec.rb +79 -0
  66. data/spec/imw/resources/archive_spec.rb +69 -0
  67. data/spec/imw/resources/archives_and_compressed/bz2_spec.rb +15 -0
  68. data/spec/imw/resources/archives_and_compressed/gz_spec.rb +15 -0
  69. data/spec/imw/resources/archives_and_compressed/rar_spec.rb +16 -0
  70. data/spec/imw/resources/archives_and_compressed/tar_spec.rb +16 -0
  71. data/spec/imw/resources/archives_and_compressed/tarbz2_spec.rb +24 -0
  72. data/spec/imw/resources/archives_and_compressed/targz_spec.rb +21 -0
  73. data/spec/imw/resources/archives_and_compressed/zip_spec.rb +16 -0
  74. data/spec/imw/resources/compressed_file_spec.rb +48 -0
  75. data/spec/imw/resources/compressible_spec.rb +36 -0
  76. data/spec/imw/resources/formats/delimited_spec.rb +33 -0
  77. data/spec/imw/resources/formats/json_spec.rb +32 -0
  78. data/spec/imw/resources/formats/sgml_spec.rb +24 -0
  79. data/spec/imw/resources/formats/yaml_spec.rb +41 -0
  80. data/spec/imw/resources/local_spec.rb +98 -0
  81. data/spec/imw/resources/remote_spec.rb +35 -0
  82. data/spec/imw/resources/schemes/hdfs_spec.rb +61 -0
  83. data/spec/imw/resources/schemes/http_spec.rb +19 -0
  84. data/spec/imw/resources/schemes/s3_spec.rb +19 -0
  85. data/spec/imw/transforms/archiver_spec.rb +120 -0
  86. data/spec/imw/transforms/transferer_spec.rb +113 -0
  87. data/spec/imw/utils/paths_spec.rb +5 -33
  88. data/spec/imw/utils/shared_paths_spec.rb +29 -0
  89. data/spec/spec_helper.rb +5 -5
  90. data/spec/support/paths_matcher.rb +67 -0
  91. data/spec/support/random.rb +39 -36
  92. metadata +88 -75
  93. data/lib/imw/dataset/task.rb +0 -41
  94. data/lib/imw/files/archive.rb +0 -113
  95. data/lib/imw/files/basicfile.rb +0 -122
  96. data/lib/imw/files/binary.rb +0 -28
  97. data/lib/imw/files/compressed_file.rb +0 -93
  98. data/lib/imw/files/compressed_files_and_archives.rb +0 -334
  99. data/lib/imw/files/compressible.rb +0 -103
  100. data/lib/imw/files/csv.rb +0 -113
  101. data/lib/imw/files/directory.rb +0 -62
  102. data/lib/imw/files/excel.rb +0 -84
  103. data/lib/imw/files/json.rb +0 -41
  104. data/lib/imw/files/sgml.rb +0 -46
  105. data/lib/imw/files/text.rb +0 -68
  106. data/lib/imw/files/yaml.rb +0 -46
  107. data/lib/imw/files.rb +0 -125
  108. data/lib/imw/packagers/archiver.rb +0 -126
  109. data/lib/imw/packagers/s3_mover.rb +0 -36
  110. data/lib/imw/packagers.rb +0 -8
  111. data/lib/imw/utils/components.rb +0 -61
  112. data/lib/imw/utils/config.rb +0 -46
  113. data/lib/imw/utils/extensions/class/attribute_accessors.rb +0 -8
  114. data/lib/imw/utils/extensions/core.rb +0 -27
  115. data/lib/imw/utils/extensions/dir.rb +0 -24
  116. data/lib/imw/utils/extensions/file_core.rb +0 -64
  117. data/lib/imw/utils/extensions/typed_struct.rb +0 -22
  118. data/lib/imw/utils/extensions/uri.rb +0 -59
  119. data/lib/imw/utils/view/dump_csv.rb +0 -112
  120. data/lib/imw/utils/view/dump_csv_older.rb +0 -117
  121. data/lib/imw/utils/view.rb +0 -113
  122. data/spec/imw/dataset/datamapper/uri_spec.rb +0 -43
  123. data/spec/imw/dataset/datamapper_spec_helper.rb +0 -11
  124. data/spec/imw/files/archive_spec.rb +0 -118
  125. data/spec/imw/files/basicfile_spec.rb +0 -121
  126. data/spec/imw/files/bz2_spec.rb +0 -32
  127. data/spec/imw/files/compressed_file_spec.rb +0 -96
  128. data/spec/imw/files/compressible_spec.rb +0 -100
  129. data/spec/imw/files/file_spec.rb +0 -144
  130. data/spec/imw/files/gz_spec.rb +0 -32
  131. data/spec/imw/files/rar_spec.rb +0 -33
  132. data/spec/imw/files/tar_spec.rb +0 -31
  133. data/spec/imw/files/text_spec.rb +0 -23
  134. data/spec/imw/files/zip_spec.rb +0 -31
  135. data/spec/imw/files_spec.rb +0 -38
  136. data/spec/imw/packagers/archiver_spec.rb +0 -125
  137. data/spec/imw/packagers/s3_mover_spec.rb +0 -7
  138. data/spec/imw/utils/extensions/file_core_spec.rb +0 -72
  139. data/spec/imw/utils/extensions/find_spec.rb +0 -113
  140. data/spec/imw/workflow/rip/local_spec.rb +0 -89
  141. data/spec/imw/workflow/rip_spec.rb +0 -27
  142. data/spec/support/archive_contents_matcher.rb +0 -94
  143. data/spec/support/directory_contents_matcher.rb +0 -61
@@ -0,0 +1,61 @@
1
+ require File.join(File.dirname(__FILE__),'../../../spec_helper')
2
+
3
+ describe IMW::Resources::Schemes::HDFS do
4
+ before do
5
+ def fake_hdfs_resource path, num_dirs=nil, num_files=nil, size=nil
6
+ if num_dirs && num_files && size
7
+ response = " #{num_dirs} #{num_files} #{size} hdfs://localhost#{path}"
8
+ else
9
+ response = ""
10
+ end
11
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:count, path).and_return(response)
12
+ IMW.open("hdfs://#{path}")
13
+ end
14
+ @path = '/path/to/myfile'
15
+ end
16
+
17
+ describe "refreshing its properties" do
18
+ it "should correctly get properties for a resource which exists" do
19
+ resource = fake_hdfs_resource(@path, 2, 3, 1000)
20
+ resource.exist?.should be_true
21
+ resource.num_dirs.should == 2
22
+ resource.num_files.should == 3
23
+ resource.size.should == 1000
24
+ end
25
+
26
+ it "should gracefully handle a resource which doesn't exist" do
27
+ resource = fake_hdfs_resource(@path)
28
+ resource.exist?.should be_false
29
+ end
30
+
31
+ it "should execute the correct command to delete the path" do
32
+ resource = fake_hdfs_resource(@path, 2, 3, 1000)
33
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:rm, resource.path)
34
+ resource.rm
35
+ end
36
+
37
+ it "should execute the correct command to delete the path when skipping the trash" do
38
+ resource = fake_hdfs_resource(@path, 2, 3, 1000)
39
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:rm, '-skipTrash', resource.path)
40
+ resource.rm :skip_trash => true
41
+ end
42
+
43
+ it "should recognize a file and extend it properly" do
44
+ resource = fake_hdfs_resource(@path, 0, 1, 1000)
45
+ resource.num_dirs.should == 0
46
+ resource.num_files.should == 1
47
+ resource.exist?.should be_true
48
+ resource.is_directory?.should be_false
49
+ resource.resource_modules.should include(IMW::Resources::Schemes::HDFSFile)
50
+ end
51
+
52
+ it "should recognize a directory and extend it properly" do
53
+ resource = fake_hdfs_resource(@path, 2, 1, 1000)
54
+ resource.num_dirs.should == 2
55
+ resource.num_files.should == 1
56
+ resource.exist?.should be_true
57
+ resource.is_directory?.should be_true
58
+ resource.resource_modules.should include(IMW::Resources::Schemes::HDFSDirectory)
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,19 @@
1
+ require File.join(File.dirname(__FILE__),'../../../spec_helper')
2
+
3
+ describe IMW::Resources::Schemes::HTTP do
4
+
5
+ describe "finding its effective basename" do
6
+ it "should use the real basename when present" do
7
+ IMW.open('http://www.google.com/foobar').effective_basename.should == 'foobar'
8
+ end
9
+
10
+ it "should use '_index' when at the root (without a slash)" do
11
+ IMW.open('http://www.google.com').effective_basename.should == '_index'
12
+ end
13
+
14
+ it "should use '_index' when at the root (even when a slash is given)" do
15
+ IMW.open('http://www.google.com/').effective_basename.should == '_index'
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,19 @@
1
+ require File.join(File.dirname(__FILE__),'../../../spec_helper')
2
+
3
+ describe IMW::Resources::Schemes::S3 do
4
+
5
+ describe 'manipulating S3 paths' do
6
+ before do
7
+ @resource = IMW::Resource.new('s3://mybucket/foobar/foo.txt')
8
+ end
9
+
10
+ it "should set the bucket" do
11
+ @resource.bucket.should == 'mybucket'
12
+ end
13
+
14
+ it "can generate an S3N url" do
15
+ @resource.s3n_url.should == 's3n://mybucket/foobar/foo.txt'
16
+ end
17
+ end
18
+
19
+ end
@@ -0,0 +1,120 @@
1
+ require File.dirname(__FILE__) + "/../../spec_helper"
2
+
3
+ describe IMW::Transforms::Archiver do
4
+ before do
5
+ @name = 'foobar'
6
+
7
+ # remote files
8
+ @homepage = "http://www.google.com"
9
+ @website = "http://www.google.com/support/"
10
+ @remote_files = [@homepage, @website]
11
+
12
+ # regular files
13
+ @csv = "foobar-csv.csv"
14
+ @xml = "foobar-xml.xml"
15
+ @txt = "foobar-txt.txt"
16
+ @blah = "foobar"
17
+ @files = [@csv, @xml, @txt, @blah]
18
+
19
+ # compressed files
20
+ @bz2 = "foobar-bz2.bz2"
21
+ @gz = "foobar-gz.gz"
22
+ @compressed_files = [@bz2, @gz]
23
+
24
+ # archives
25
+ @zip = "foobar-zip.zip"
26
+ @tarbz2 = "foobar-tarbz2.tar.bz2"
27
+ @targz = "foobar-targz.tar.gz"
28
+ @tar = "foobar-tar.tar"
29
+ @rar = "foobar-rar.rar"
30
+ @archives = [@zip, @tarbz2, @targz, @rar, @tar]
31
+
32
+ @local_files = @files + @compressed_files + @archives
33
+
34
+ @all_files = @remote_files + @local_files
35
+
36
+ @local_files.each do |path|
37
+ IMWTest::Random.file path
38
+ end
39
+
40
+ @archiver = IMW::Transforms::Archiver.new @name, @all_files
41
+ end
42
+
43
+ after do
44
+ @archiver.clean!
45
+ end
46
+
47
+ describe "preparing input files" do
48
+
49
+ describe "before preparing input files" do
50
+ it "should not be prepared when initialized" do
51
+ @archiver.prepared?.should be_false
52
+ end
53
+ end
54
+
55
+ describe "after preparing files" do
56
+ before { @archiver.prepare! }
57
+
58
+ it "should be prepared" do
59
+ @archiver.prepared?.should be_true
60
+ end
61
+
62
+ it "should name its archive directory properly" do
63
+ @archiver.tmp_dir.should contain(@name)
64
+ end
65
+
66
+ it "should copy regular files to its archive directory" do
67
+ @archiver.dir.should contain(*@files)
68
+ @local_files.each { |path| IMW.open(path).exist?.should be_true }
69
+ end
70
+
71
+ it "should copy remote files to its archive directory" do
72
+ @archiver.dir.should contain('_index', 'support') # _index from Http#effective_basename on http://www.google.com
73
+ end
74
+
75
+ it "should uncompress compressed files to its archive directory" do
76
+ @archiver.dir.should contain('foobar-bz2', 'foobar-gz')
77
+ @archiver.dir.should_not contain(*@compressed_files)
78
+ end
79
+
80
+ it "should copy the content of archive files to its archive directory (but not the actual archives)" do
81
+ @archives.each do |archive|
82
+ @archiver.dir.should_not contain(archive)
83
+ @archiver.dir.should contain(*IMW.open(archive).contents)
84
+ end
85
+ end
86
+
87
+ end
88
+ end
89
+
90
+ describe "when packaging files" do
91
+ @packages = ["package.tar.bz2", "package.zip", "package.tar.gz", "package.tar", "package.rar"]
92
+
93
+ @packages.each do |package|
94
+ it "should create a #{package} file containing all the files and return it" do
95
+ output = @archiver.package!(package)
96
+ output.basename.should == package
97
+ @archiver.tmp_dir.should contain(IMW.open(package).contents)
98
+ end
99
+ end
100
+
101
+ describe 'when packaging into multiple output formats' do
102
+
103
+ it "should prepare input files without being asked" do
104
+ @archiver.prepared?.should be_false
105
+ @archiver.package! 'package.tar.bz2'
106
+ @archiver.prepared?.should be_true
107
+ end
108
+
109
+ it "should not prepare input files once they've already been prepared" do
110
+ @archiver.prepared?.should be_false
111
+ @archiver.package! 'package.tar.bz2'
112
+ @archiver.prepared?.should be_true
113
+ @archiver.should_not_receive(:prepare!)
114
+ @archiver.package! 'package.tar.gz'
115
+ end
116
+ end
117
+ end
118
+ end
119
+
120
+
@@ -0,0 +1,113 @@
1
+ require File.dirname(__FILE__) + "/../../spec_helper"
2
+
3
+ describe IMW::Transforms::Transferer do
4
+ before do
5
+ @local = IMW.open("foobar.txt")
6
+ @http = IMW.open("http://www.google.com")
7
+ @hdfs = IMW.open("hdfs:///path/to/foobar.txt")
8
+ @s3 = IMW.open("s3://mybucket/foo/bar")
9
+ end
10
+
11
+ it "should raise an error unless the action is one of :cp, :copy, :mv :move, or :mv!" do
12
+ IMW::Transforms::Transferer.new(:cp, @local, @http).should be
13
+ IMW::Transforms::Transferer.new(:copy, @local, @http).should be
14
+ IMW::Transforms::Transferer.new(:mv, @local, @http).should be
15
+ IMW::Transforms::Transferer.new(:move, @local, @http).should be
16
+ IMW::Transforms::Transferer.new(:mv!, @local, @http).should be
17
+ lambda { IMW::Transforms::Transferer.new(:foobar, @local, @http) }.should raise_error(IMW::ArgumentError)
18
+ end
19
+
20
+ it "should raise an error if the source and the destination have the same URI" do
21
+ lambda { IMW::Transforms::Transferer.new(:cp, @local, @local) }.should raise_error(IMW::PathError)
22
+ end
23
+
24
+ describe "transfering local files" do
25
+ it "should raise an error if the source doesn't exist" do
26
+ lambda { IMW::Transforms::Transferer.new(:cp, @local, 'barbaz.txt').transfer! }.should raise_error(IMW::PathError)
27
+ end
28
+
29
+ it "can copy a local file" do
30
+ IMWTest::Random.file @local.path
31
+ IMW::Transforms::Transferer.new(:cp, @local, 'barbaz.txt').transfer!
32
+ @local.exist?.should be_true
33
+ IMW.open('barbaz.txt').exist?.should be_true
34
+ end
35
+
36
+ it "can move a local file" do
37
+ IMWTest::Random.file @local.path
38
+ IMW::Transforms::Transferer.new(:mv, @local, 'barbaz.txt').transfer!
39
+ @local.exist?.should be_false
40
+ IMW.open('barbaz.txt').exist?.should be_true
41
+ end
42
+
43
+ end
44
+
45
+ describe "transferring HTTP files" do
46
+ it "can copy a remote file to a local path" do
47
+ IMW::Transforms::Transferer.new(:cp, @http, @local).transfer!
48
+ @local.exist?.should be_true
49
+ end
50
+ end
51
+
52
+ describe "transferring S3 files" do
53
+
54
+ it "can copy an S3 file to a local path" do
55
+ IMW::Resources::Schemes::S3.should_receive(:get).with(@s3, @local)
56
+ IMW::Transforms::Transferer.new(:cp, @s3, @local).transfer!
57
+ end
58
+
59
+ it "can copy a local path to an S3 file" do
60
+ IMWTest::Random.file @local.path
61
+ IMW::Resources::Schemes::S3.should_receive(:put).with(@local, @s3)
62
+ IMW::Transforms::Transferer.new(:cp, @local, @s3).transfer!
63
+ end
64
+
65
+ it "can copy between S3 files" do
66
+ @new_s3 = IMW.open('s3://mybucket/new/path')
67
+ IMW::Resources::Schemes::S3.should_receive(:copy).with(@s3, @new_s3)
68
+ IMW::Transforms::Transferer.new(:cp, @s3, @new_s3).transfer!
69
+ end
70
+ end
71
+
72
+ describe "transferring HDFS files" do
73
+ before do
74
+ IMW::Resources::Schemes::HDFS.stub!(:fs)
75
+ end
76
+
77
+ it "can copy a local file to an HDFS path" do
78
+ IMWTest::Random.file @local.path
79
+
80
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:put, @local.path, @hdfs.path)
81
+ IMW::Transforms::Transferer.new(:cp, @local, @hdfs).transfer!
82
+ end
83
+
84
+ it "can copy an HDFS file to a local path" do
85
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:get, @hdfs.path, @local.path)
86
+ IMW::Transforms::Transferer.new(:cp, @hdfs, @local).transfer!
87
+ end
88
+
89
+ it "can copy between HDFS paths" do
90
+ @new_hdfs = IMW.open('hdfs:///a/new/path')
91
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @new_hdfs.path)
92
+ IMW::Transforms::Transferer.new(:cp, @hdfs, @new_hdfs).transfer!
93
+ end
94
+
95
+ it "can move between HDFS paths" do
96
+ @new_hdfs = IMW.open('hdfs:///a/new/path')
97
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:mv, @hdfs.path, @new_hdfs.path)
98
+ IMW::Transforms::Transferer.new(:mv, @hdfs, @new_hdfs).transfer!
99
+ end
100
+
101
+ it "can copy from S3 to HDFS" do
102
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:cp, @s3.s3n_url, @hdfs.path)
103
+ IMW::Transforms::Transferer.new(:cp, @s3, @hdfs).transfer!
104
+ end
105
+
106
+ it "can copy from HDFS to S3" do
107
+ IMW::Resources::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @s3.s3n_url)
108
+ IMW::Transforms::Transferer.new(:cp, @hdfs, @s3).transfer!
109
+ end
110
+ end
111
+ end
112
+
113
+
@@ -1,38 +1,10 @@
1
1
  require File.join(File.dirname(__FILE__),'../../spec_helper')
2
- require 'imw'
3
- require 'imw/utils/paths'
2
+ require File.join(File.dirname(__FILE__), '/shared_paths_spec')
4
3
 
5
4
  describe IMW do
6
- include IMW
7
- before(:each) do
8
- IMW::PATHS = {
9
- :data => '/data',
10
- :weather => 'ftp.ncdc.noaa.gov/pub/data/noaa',
11
- :first => ['1', :second, 'last'],
12
- :second => ['2', :third],
13
- :third => ['3'],
14
- }
5
+ before do
6
+ @path_manager = IMW
15
7
  end
16
-
17
- it 'is idempotent on a string' do
18
- path_to('hi').should == 'hi'
19
- end
20
-
21
- it 'has an absolute path to the data dir' do
22
- path_to(:data).should =~ %r{^/}
23
- end
24
-
25
- it 'handles mixed array and sym args' do
26
- path_to( [:data, 'hi'], [[['there']]]).should == '/data/hi/there'
27
- end
28
-
29
- it 'expands to later generations' do
30
- path_to(:first).should == File.join('1/2/3/last')
31
- end
32
-
33
- it 'expands interior symbols' do
34
- path_to(['hadoop1:/working', :data, :weather]).should ==
35
- File.join('hadoop1:/working/data/ftp.ncdc.noaa.gov/pub/data/noaa')
36
- end
37
-
8
+ it_should_behave_like "an object that manages paths"
38
9
  end
10
+
@@ -0,0 +1,29 @@
1
+ require File.join(File.dirname(__FILE__),'../../spec_helper')
2
+
3
+ share_examples_for "an object that manages paths" do
4
+ before do
5
+ @path_manager.add_path :testing, '/testing'
6
+ @path_manager.add_path :first, '/1'
7
+ end
8
+
9
+ it 'returns a string when given a string' do
10
+ @path_manager.path_to('hi').should == 'hi'
11
+ end
12
+
13
+ it 'returns a path when given a registered symbol' do
14
+ @path_manager.path_to(:testing).should == '/testing'
15
+ end
16
+
17
+ it 'raises an error when given a unregistered symbol' do
18
+ lambda { @path_manager.path_to(:foobar) }.should raise_error(IMW::PathError)
19
+ end
20
+
21
+ it 'returns a constructed path when passed a mixture of symbols, strings, and arrays ' do
22
+ @path_manager.path_to( [:testing, 'hi'], [[['there']]]).should == '/testing/hi/there'
23
+ end
24
+
25
+ it 'will correctly expand paths themselves defined via symbols' do
26
+ @path_manager.add_path(:first, :testing, '1')
27
+ @path_manager.path_to(:first).should == '/testing/1'
28
+ end
29
+ end
data/spec/spec_helper.rb CHANGED
@@ -1,23 +1,23 @@
1
1
  IMW_ROOT_DIR = File.join(File.expand_path(File.dirname(__FILE__)), '..') unless defined? IMW_ROOT_DIR
2
- IMW_SPEC_DIR = File.join(IMW_ROOT_DIR, 'spec') unless defined? IMW_SPEC_DIR
3
- IMW_LIB_DIR = File.join(IMW_ROOT_DIR, 'lib') unless defined? IMW_LIB_DIR
2
+ IMW_SPEC_DIR = File.join(IMW_ROOT_DIR, 'spec') unless defined? IMW_SPEC_DIR
3
+ IMW_LIB_DIR = File.join(IMW_ROOT_DIR, 'lib') unless defined? IMW_LIB_DIR
4
4
  $: << IMW_LIB_DIR
5
5
 
6
6
  require 'rubygems'
7
7
  require 'spec'
8
- require 'fileutils'
9
8
  require 'imw'
10
9
 
11
10
  Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |path| require path }
12
11
 
13
12
  module IMWTest
14
- TMP_DIR = "/tmp/imwtest"
13
+ TMP_DIR = "/tmp/imwtest" unless defined?(TMP_DIR)
14
+ DATA_DIR = File.join(IMW_SPEC_DIR, 'data') unless defined?(DATA_DIR)
15
15
  end
16
16
 
17
17
  Spec::Runner.configure do |config|
18
18
 
19
19
  config.include CustomMatchers
20
-
20
+
21
21
  config.before do
22
22
  FileUtils.mkdir_p IMWTest::TMP_DIR
23
23
  FileUtils.cd IMWTest::TMP_DIR
@@ -0,0 +1,67 @@
1
+ require 'set'
2
+
3
+ module Spec
4
+ module Matchers
5
+ module IMW
6
+
7
+ class PathsMatcher
8
+
9
+ attr_accessor :given, :given_contents, :given_base, :to_match, :to_match_contents, :to_match_base
10
+
11
+ def initialize given, options={}
12
+ @given_base = options[:given_base] || options[:relative_to]
13
+ @to_match_base = options[:to_match_base]
14
+ @given = given
15
+ @given_contents = get_contents(given, given_base)
16
+ end
17
+
18
+ def matches? to_match
19
+ @to_match = to_match
20
+ @to_match_contents = get_contents(to_match, to_match_base)
21
+ to_match_contents == given_contents
22
+ end
23
+
24
+ def failure_message
25
+ given_string = given_contents.to_a.join("\n\t")
26
+ to_match_string = to_match_contents.to_a.join("\n\t")
27
+ "expected contents to be identical.\n\ngiven #{given.inspect}:\n\t#{given_string}\n\nto match #{to_match}:\n\t#{to_match_string}"
28
+ end
29
+
30
+ def negative_failure_message
31
+ "expected contents of #{given} and #{to_match} to be different"
32
+ end
33
+
34
+ protected
35
+ def get_contents obj, base=nil
36
+ if obj.is_a?(String) || obj.is_a?(Array)
37
+ contents = [obj].flatten.map do |raw_path|
38
+ path = File.expand_path(raw_path)
39
+ if File.directory?(path)
40
+ Dir[path + "/**/*"]
41
+ else
42
+ path
43
+ end
44
+ end.flatten
45
+ else
46
+ # obj is an IMW obj (archive or directory) so it has a
47
+ # contents method
48
+ contents = obj.contents
49
+ end
50
+ if base
51
+ contents.map do |path|
52
+ new_path = path[base.length + 1..-1]
53
+ new_path = nil if !new_path.nil? && new_path.size == 0
54
+ new_path
55
+ end.compact.to_set
56
+ else
57
+ contents.to_set
58
+ end
59
+ end
60
+ end
61
+
62
+ def contain_paths_like given, options={}
63
+ PathsMatcher.new(given, options)
64
+ end
65
+ end
66
+ end
67
+ end
@@ -3,36 +3,23 @@ require 'fileutils'
3
3
  module IMWTest
4
4
  module Random
5
5
 
6
- STRING_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + [' ',' ',' ',' ',' ']
7
- TEXT_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + [' ',' ',' ',' ',' ',"\n"]
8
- FILENAME_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + ["-","_"]
9
- FILENAME_MAX_LENGTH = 9
10
- TEXT_MAX_LENGTH = 1024
11
- EXTENSIONS = {
12
- /\.csv$/ => :csv_file,
13
- /\.xml$/ => :xml_file,
14
- /\.html$/ => :html_file,
15
- /\.tar$/ => :tar_file,
16
- /\.tar\.gz$/ => :targz_file,
17
- /\.tar\.bz2$/ => :tarbz2_file,
18
- /\.rar$/ => :rar_file,
19
- /\.zip$/ => :zip_file
20
- }
21
- EXTERNAL_PROGRAMS = if defined?(IMW) && defined?(IMW::EXTERNAL_PROGRAMS)
22
- IMW::EXTERNAL_PROGRAMS
23
- else
24
- {
25
- :tar => "tar",
26
- :rar => "rar",
27
- :zip => "zip",
28
- :unzip => "unzip",
29
- :gzip => "gzip",
30
- :bzip2 => "bzip2",
31
- :wget => "wget"
32
- }
33
- end
34
-
35
- private
6
+ STRING_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + [' ',' ',' ',' ',' '] unless defined?(STRING_CHARS)
7
+ TEXT_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + [' ',' ',' ',' ',' ',"\n"] unless defined?(TEXT_CHARS)
8
+ FILENAME_CHARS = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + ["-","_",' '] unless defined?(FILENAME_CHARS)
9
+ FILENAME_MAX_LENGTH = 9 unless defined?(FILENAME_MAX_LENGTH)
10
+ TEXT_MAX_LENGTH = 1024 unless defined?(TEXT_MAX_LENGTH)
11
+ EXTENSIONS = [
12
+ [/\.csv$/ , :csv_file],
13
+ [/\.xml$/ , :xml_file],
14
+ [/\.html$/ , :html_file],
15
+ [/\.tar\.gz$/ , :targz_file],
16
+ [/\.tar\.bz2$/ , :tarbz2_file],
17
+ [/\.bz2$/ , :bz2_file],
18
+ [/\.gz$/ , :gz_file],
19
+ [/\.tar$/ , :tar_file],
20
+ [/\.rar$/ , :rar_file],
21
+ [/\.zip$/ , :zip_file]
22
+ ] unless defined?(EXTENSIONS)
36
23
  # Return a random filename. Optional +length+ to set the maximum
37
24
  # length of the filename returned.
38
25
  def self.basename options = {}
@@ -54,7 +41,6 @@ module IMWTest
54
41
  (1..length).map { |i| char_pool.random }.join
55
42
  end
56
43
 
57
- public
58
44
  # Create a random file by matching the extension of the given
59
45
  # +filename+ or a text file if no match is found.
60
46
  def self.file filename
@@ -115,7 +101,7 @@ module IMWTest
115
101
  def self.tar_file filename
116
102
  tmpd = File.dirname(filename) + '/dir'
117
103
  directory_with_files(tmpd)
118
- FileUtils.cd(tmpd) {|dir| system("#{EXTERNAL_PROGRAMS[:tar]} -cf file.tar *") }
104
+ FileUtils.cd(tmpd) {|dir| system("tar -cf file.tar *") }
119
105
  FileUtils.cp(tmpd + "/file.tar",filename)
120
106
  FileUtils.rm_rf(tmpd)
121
107
  end
@@ -126,7 +112,7 @@ module IMWTest
126
112
  tar = File.dirname(filename) + "/file.tar"
127
113
  targz = tar + ".gz"
128
114
  tar_file tar
129
- system("#{EXTERNAL_PROGRAMS[:gzip]} #{tar}")
115
+ system("gzip #{tar}")
130
116
  FileUtils.cp(targz,filename)
131
117
  FileUtils.rm(targz)
132
118
  end
@@ -137,17 +123,34 @@ module IMWTest
137
123
  tar = File.dirname(filename) + "/file.tar"
138
124
  tarbz2 = tar + ".bz2"
139
125
  tar_file tar
140
- system("#{EXTERNAL_PROGRAMS[:bzip2]} #{tar}")
126
+ system("bzip2 #{tar}")
141
127
  FileUtils.cp(tarbz2,filename)
142
128
  FileUtils.rm(tarbz2)
143
129
  end
144
130
 
131
+ # Create a .bz2 file at the given +filename+.
132
+ def self.bz2_file filename
133
+ text_path = File.dirname(filename) + "/fake_file"
134
+ text_file(text_path)
135
+ system("bzip2 #{text_path}")
136
+ FileUtils.mv(text_path + ".bz2", filename)
137
+ end
138
+
139
+ # Create a .gz file at the given +filename+.
140
+ def self.gz_file filename
141
+ text_path = File.dirname(filename) + "/fake_file"
142
+ text_file(text_path)
143
+ system("gzip #{text_path}")
144
+ FileUtils.mv(text_path + ".gz", filename)
145
+ end
146
+
147
+
145
148
  # Create a compressed rar archive at the given +filename+
146
149
  # containing random files.
147
150
  def self.rar_file filename
148
151
  tmpd = File.dirname(filename) + '/dir'
149
152
  directory_with_files(tmpd)
150
- FileUtils.cd(tmpd) {|dir| system("#{EXTERNAL_PROGRAMS[:rar]} a -r -o+ file.rar *") }
153
+ FileUtils.cd(tmpd) {|dir| system("rar a -o+ -inul file.rar *") }
151
154
  FileUtils.cp(tmpd + "/file.rar",filename)
152
155
  FileUtils.rm_rf(tmpd)
153
156
  end
@@ -157,7 +160,7 @@ module IMWTest
157
160
  def self.zip_file filename
158
161
  tmpd = File.dirname(filename) + '/dir'
159
162
  directory_with_files(tmpd)
160
- FileUtils.cd(tmpd) {|dir| system("#{EXTERNAL_PROGRAMS[:zip]} -r file.zip *") }
163
+ FileUtils.cd(tmpd) {|dir| system("zip -qqr file.zip *") }
161
164
  FileUtils.cp(tmpd + "/file.zip",filename)
162
165
  FileUtils.rm_rf(tmpd)
163
166
  end