remote_table 0.2.16 → 0.2.17
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/remote_table/file.rb +4 -4
- data/lib/remote_table/package.rb +2 -1
- data/lib/remote_table/request.rb +6 -8
- data/lib/remote_table.rb +20 -0
- data/remote_table.gemspec +2 -2
- data/test/remote_table_test.rb +65 -58
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.17
|
data/lib/remote_table/file.rb
CHANGED
@@ -59,13 +59,13 @@ class RemoteTable
|
|
59
59
|
|
60
60
|
def skip_rows!
|
61
61
|
return unless skip
|
62
|
-
|
62
|
+
RemoteTable.backtick_with_reporting "cat #{path} | tail -n +#{skip + 1} > #{path}.tmp"
|
63
63
|
FileUtils.mv "#{path}.tmp", path
|
64
64
|
end
|
65
65
|
|
66
66
|
def convert_file_to_utf8!
|
67
67
|
return if encoding == 'UTF-8' or encoding == 'UTF8'
|
68
|
-
|
68
|
+
RemoteTable.backtick_with_reporting "iconv -c -f #{encoding} -t UTF-8 #{path} > #{path}.tmp"
|
69
69
|
FileUtils.mv "#{path}.tmp", path
|
70
70
|
end
|
71
71
|
|
@@ -75,13 +75,13 @@ class RemoteTable
|
|
75
75
|
|
76
76
|
def cut_columns!
|
77
77
|
return unless cut
|
78
|
-
|
78
|
+
RemoteTable.backtick_with_reporting "cat #{path} | cut -c #{cut} > #{path}.tmp"
|
79
79
|
FileUtils.mv "#{path}.tmp", path
|
80
80
|
end
|
81
81
|
|
82
82
|
def crop_rows!
|
83
83
|
return unless crop
|
84
|
-
|
84
|
+
RemoteTable.backtick_with_reporting "cat #{path} | tail -n +#{crop.first} | head -n #{crop.last - crop.first + 1} > #{path}.tmp"
|
85
85
|
FileUtils.mv "#{path}.tmp", path
|
86
86
|
end
|
87
87
|
|
data/lib/remote_table/package.rb
CHANGED
@@ -47,7 +47,8 @@ class RemoteTable
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def move_and_process(path, extname, cmd, args)
|
50
|
-
|
50
|
+
FileUtils.mv path, "#{path}.#{extname}"
|
51
|
+
RemoteTable.backtick_with_reporting "#{cmd} #{path}.#{extname} #{args}"
|
51
52
|
end
|
52
53
|
|
53
54
|
# ex. A: 2007-01.csv.gz (compression not capable of storing multiple files)
|
data/lib/remote_table/request.rb
CHANGED
@@ -20,16 +20,14 @@ class RemoteTable
|
|
20
20
|
if parsed_url.scheme == 'file'
|
21
21
|
parsed_url.path
|
22
22
|
else
|
23
|
-
|
24
|
-
curl
|
25
|
-
--
|
26
|
-
--
|
27
|
-
--
|
28
|
-
|
29
|
-
"#{parsed_url}" \
|
23
|
+
RemoteTable.backtick_with_reporting %{
|
24
|
+
curl
|
25
|
+
--header "Expect: "
|
26
|
+
--location
|
27
|
+
#{"--data \"#{form_data}\"" if form_data.present?}
|
28
|
+
"#{parsed_url}"
|
30
29
|
--output "#{path}"
|
31
30
|
}
|
32
|
-
`#{cmd}`
|
33
31
|
path
|
34
32
|
end
|
35
33
|
end
|
data/lib/remote_table.rb
CHANGED
@@ -56,6 +56,26 @@ class RemoteTable
|
|
56
56
|
def <=>(other)
|
57
57
|
raise "Not implemented"
|
58
58
|
end
|
59
|
+
|
60
|
+
protected
|
61
|
+
|
62
|
+
# TODO this should probably live somewhere else
|
63
|
+
def self.backtick_with_reporting(cmd)
|
64
|
+
cmd = cmd.gsub /\s+/m, ' '
|
65
|
+
cmd = cmd + ' 2>&1' unless cmd.include? '2>'
|
66
|
+
output = `#{cmd}`
|
67
|
+
unless $?.success?
|
68
|
+
raise %{
|
69
|
+
From the remote_table gem...
|
70
|
+
|
71
|
+
Command failed:
|
72
|
+
#{cmd}
|
73
|
+
|
74
|
+
Output:
|
75
|
+
#{output}
|
76
|
+
}
|
77
|
+
end
|
78
|
+
end
|
59
79
|
|
60
80
|
private
|
61
81
|
|
data/remote_table.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{remote_table}
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.17"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
|
12
|
-
s.date = %q{2010-04-
|
12
|
+
s.date = %q{2010-04-28}
|
13
13
|
s.description = %q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
|
14
14
|
s.email = %q{seamus@abshere.net}
|
15
15
|
s.extra_rdoc_files = [
|
data/test/remote_table_test.rb
CHANGED
@@ -50,21 +50,7 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
50
50
|
]
|
51
51
|
end
|
52
52
|
|
53
|
-
if ENV['
|
54
|
-
end
|
55
|
-
|
56
|
-
if ENV['OLD'] == 'true'
|
57
|
-
should "read an HTML table made with frontpage" do
|
58
|
-
t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-E.htm",
|
59
|
-
:encoding => 'US-ASCII',
|
60
|
-
:row_xpath => '//table/tr[2]/td/table/tr',
|
61
|
-
:column_xpath => 'td'
|
62
|
-
assert_equal 'E110', t.rows.first['Designator']
|
63
|
-
assert_equal 'EMBRAER', t.rows.first['Manufacturer']
|
64
|
-
assert_equal 'EZKC', t.rows.last['Designator']
|
65
|
-
assert_equal 'EZ King Cobra', t.rows.last['Model']
|
66
|
-
end
|
67
|
-
|
53
|
+
if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
|
68
54
|
should "open an XLS inside a zip file" do
|
69
55
|
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls')
|
70
56
|
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
@@ -79,6 +65,58 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
79
65
|
assert_equal nil, t.rows.first[:Manufacturer]
|
80
66
|
end
|
81
67
|
|
68
|
+
should "open a CSV inside a zip file" do
|
69
|
+
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
|
70
|
+
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
71
|
+
assert_equal 'NSX', t.rows.first['carline name']
|
72
|
+
assert_equal 'TOYOTA', t.rows.last['Manufacturer']
|
73
|
+
assert_equal 'RAV4 SOFT TOP 4WD', t.rows.last['carline name']
|
74
|
+
end
|
75
|
+
|
76
|
+
should "open a fixed-width file with an inline schema inside a zip file" do
|
77
|
+
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
78
|
+
:filename => 'Gd6-dsc.txt',
|
79
|
+
:format => :fixed_width,
|
80
|
+
:crop => 21..26, # inclusive
|
81
|
+
:cut => '2-',
|
82
|
+
:select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
83
|
+
:schema => [[ 'code', 2, { :type => :string } ],
|
84
|
+
[ 'spacer', 2 ],
|
85
|
+
[ 'name', 52, { :type => :string } ]])
|
86
|
+
assert_equal 'regular grade gasoline (octane number of 87)', t.rows.first['name']
|
87
|
+
assert_equal 'R', t.rows.first['code']
|
88
|
+
assert_equal 'electricity', t.rows.last['name']
|
89
|
+
assert_equal 'El', t.rows.last['code']
|
90
|
+
end
|
91
|
+
|
92
|
+
should "send form data, follow redirects and use a filename glob" do
|
93
|
+
url = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
|
94
|
+
form_data = 'UserTableName=T_100_Segment__All_Carriers&DBShortName=Air_Carriers&RawDataTable=T_T100_SEGMENT_ALL_CARRIER&sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D1+AND+YEAR%3D2008&varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&grouplist=&suml=&sumRegion=&filter1=title%3D&filter2=title%3D&geo=All%A0&time=January&timename=Month&GEOGRAPHY=All&XYEAR=2008&FREQUENCY=1&AllVars=All&VarName=DEPARTURES_SCHEDULED&VarDesc=DepScheduled&VarType=Num&VarName=DEPARTURES_PERFORMED&VarDesc=DepPerformed&VarType=Num&VarName=PAYLOAD&VarDesc=Payload&VarType=Num&VarName=SEATS&VarDesc=Seats&VarType=Num&VarName=PASSENGERS&VarDesc=Passengers&VarType=Num&VarName=FREIGHT&VarDesc=Freight&VarType=Num&VarName=MAIL&VarDesc=Mail&VarType=Num&VarName=DISTANCE&VarDesc=Distance&VarType=Num&VarName=RAMP_TO_RAMP&VarDesc=RampToRamp&VarType=Num&VarName=AIR_TIME&VarDesc=AirTime&VarType=Num&VarName=UNIQUE_CARRIER&VarDesc=UniqueCarrier&VarType=Char&VarName=AIRLINE_ID&VarDesc=AirlineID&VarType=Num&VarName=UNIQUE_CARRIER_NAME&VarDesc=UniqueCarrierName&VarType=Char&VarName=UNIQUE_CARRIER_ENTITY&VarDesc=UniqCarrierEntity&VarType=Char&VarName=REGION&VarDesc=CarrierRegion&VarType=Char&VarName=CARRIER&VarDesc=Carrier&VarType=Char&VarName=CARRIER_NAME&VarDesc=CarrierName&VarType=Char&VarName=CARRIER_GROUP&VarDesc=CarrierGroup&VarType=Num&VarName=CARRIER_GROUP_NEW&VarDesc=CarrierGroupNew&VarType=Num&VarName=ORIGIN&VarDesc=Origin&VarType=Char&VarName=ORIGIN_CITY_NAME&VarDesc=OriginCityName&VarType=Char&VarName=ORIGIN_CITY_NUM&VarDesc=OriginCityNum&VarType=Num&VarName=ORIGIN_STATE_ABR&VarDesc=OriginState&VarType=Char&VarName=ORIGIN_STATE_FIPS&VarDesc=OriginStateFips&VarType=Char&VarName=ORIGIN_STATE_NM&VarDesc=OriginStateName&VarType=Char&VarName=ORIGIN_COUNTRY&VarDesc=OriginCountry&VarType=Char&VarName=ORIGIN_COUNTRY_NAME&VarDesc=OriginCountryName&VarType=Char&VarName=ORIGIN_WAC&VarDesc=OriginWac&VarType=Num&VarName=DEST&VarDesc=Dest&VarType=Char&VarName=DEST_CITY_NAME&VarDesc=DestCityName&VarType=Char&VarName=DEST_CITY_NUM&VarDesc=DestCityNum&VarType=Num&VarName=DEST_STATE_ABR&VarDesc=DestState&VarType=Char&VarName=DEST_STATE_FIPS&VarDesc=DestStateFips&VarType=Char&VarName=DEST_STATE_NM&VarDesc=DestStateName&VarType=Char&VarName=DEST_COUNTRY&VarDesc=DestCountry&VarType=Char&VarName=DEST_COUNTRY_NAME&VarDesc=DestCountryName&VarType=Char&VarName=DEST_WAC&VarDesc=DestWac&VarType=Num&VarName=AIRCRAFT_GROUP&VarDesc=AircraftGroup&VarType=Num&VarName=AIRCRAFT_TYPE&VarDesc=AircraftType&VarType=Char&VarName=AIRCRAFT_CONFIG&VarDesc=AircraftConfig&VarType=Num&VarName=YEAR&VarDesc=Year&VarType=Num&VarName=QUARTER&VarDesc=Quarter&VarType=Num&VarName=MONTH&VarDesc=Month&VarType=Num&VarName=DISTANCE_GROUP&VarDesc=DistanceGroup&VarType=Num&VarName=CLASS&VarDesc=Class&VarType=Char&VarName=DATA_SOURCE&VarDesc=DataSource&VarType=Char'
|
95
|
+
t = RemoteTable.new :url => url, :form_data => form_data, :compression => :zip, :glob => '/*.csv'
|
96
|
+
assert_equal 'United States of America', t.rows.first['DEST_COUNTRY_NAME']
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
|
101
|
+
should "connect using HTTPS if available" do
|
102
|
+
t = RemoteTable.new(:url => 'https://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA')
|
103
|
+
assert_equal 'Gulf Coast', t.rows.first['PAD district name']
|
104
|
+
assert_equal 'AL', t.rows.first['State']
|
105
|
+
assert_equal 'Rocky Mountain', t.rows.last['PAD district name']
|
106
|
+
assert_equal 'WY', t.rows.last['State']
|
107
|
+
end
|
108
|
+
|
109
|
+
should "read an HTML table made with frontpage" do
|
110
|
+
t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-E.htm",
|
111
|
+
:encoding => 'US-ASCII',
|
112
|
+
:row_xpath => '//table/tr[2]/td/table/tr',
|
113
|
+
:column_xpath => 'td'
|
114
|
+
assert_equal 'E110', t.rows.first['Designator']
|
115
|
+
assert_equal 'EMBRAER', t.rows.first['Manufacturer']
|
116
|
+
assert_equal 'EZKC', t.rows.last['Designator']
|
117
|
+
assert_equal 'EZ King Cobra', t.rows.last['Model']
|
118
|
+
end
|
119
|
+
|
82
120
|
should "hash rows without paying attention to order" do
|
83
121
|
x = ActiveSupport::OrderedHash.new
|
84
122
|
x[:a] = 1
|
@@ -116,13 +154,6 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
116
154
|
assert_equal 'Rocky Mountain', t.rows.last[4]
|
117
155
|
end
|
118
156
|
|
119
|
-
should "send form data, follow redirects and use a filename glob" do
|
120
|
-
url = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
|
121
|
-
form_data = 'UserTableName=T_100_Segment__All_Carriers&DBShortName=Air_Carriers&RawDataTable=T_T100_SEGMENT_ALL_CARRIER&sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D1+AND+YEAR%3D2008&varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&grouplist=&suml=&sumRegion=&filter1=title%3D&filter2=title%3D&geo=All%A0&time=January&timename=Month&GEOGRAPHY=All&XYEAR=2008&FREQUENCY=1&AllVars=All&VarName=DEPARTURES_SCHEDULED&VarDesc=DepScheduled&VarType=Num&VarName=DEPARTURES_PERFORMED&VarDesc=DepPerformed&VarType=Num&VarName=PAYLOAD&VarDesc=Payload&VarType=Num&VarName=SEATS&VarDesc=Seats&VarType=Num&VarName=PASSENGERS&VarDesc=Passengers&VarType=Num&VarName=FREIGHT&VarDesc=Freight&VarType=Num&VarName=MAIL&VarDesc=Mail&VarType=Num&VarName=DISTANCE&VarDesc=Distance&VarType=Num&VarName=RAMP_TO_RAMP&VarDesc=RampToRamp&VarType=Num&VarName=AIR_TIME&VarDesc=AirTime&VarType=Num&VarName=UNIQUE_CARRIER&VarDesc=UniqueCarrier&VarType=Char&VarName=AIRLINE_ID&VarDesc=AirlineID&VarType=Num&VarName=UNIQUE_CARRIER_NAME&VarDesc=UniqueCarrierName&VarType=Char&VarName=UNIQUE_CARRIER_ENTITY&VarDesc=UniqCarrierEntity&VarType=Char&VarName=REGION&VarDesc=CarrierRegion&VarType=Char&VarName=CARRIER&VarDesc=Carrier&VarType=Char&VarName=CARRIER_NAME&VarDesc=CarrierName&VarType=Char&VarName=CARRIER_GROUP&VarDesc=CarrierGroup&VarType=Num&VarName=CARRIER_GROUP_NEW&VarDesc=CarrierGroupNew&VarType=Num&VarName=ORIGIN&VarDesc=Origin&VarType=Char&VarName=ORIGIN_CITY_NAME&VarDesc=OriginCityName&VarType=Char&VarName=ORIGIN_CITY_NUM&VarDesc=OriginCityNum&VarType=Num&VarName=ORIGIN_STATE_ABR&VarDesc=OriginState&VarType=Char&VarName=ORIGIN_STATE_FIPS&VarDesc=OriginStateFips&VarType=Char&VarName=ORIGIN_STATE_NM&VarDesc=OriginStateName&VarType=Char&VarName=ORIGIN_COUNTRY&VarDesc=OriginCountry&VarType=Char&VarName=ORIGIN_COUNTRY_NAME&VarDesc=OriginCountryName&VarType=Char&VarName=ORIGIN_WAC&VarDesc=OriginWac&VarType=Num&VarName=DEST&VarDesc=Dest&VarType=Char&VarName=DEST_CITY_NAME&VarDesc=DestCityName&VarType=Char&VarName=DEST_CITY_NUM&VarDesc=DestCityNum&VarType=Num&VarName=DEST_STATE_ABR&VarDesc=DestState&VarType=Char&VarName=DEST_STATE_FIPS&VarDesc=DestStateFips&VarType=Char&VarName=DEST_STATE_NM&VarDesc=DestStateName&VarType=Char&VarName=DEST_COUNTRY&VarDesc=DestCountry&VarType=Char&VarName=DEST_COUNTRY_NAME&VarDesc=DestCountryName&VarType=Char&VarName=DEST_WAC&VarDesc=DestWac&VarType=Num&VarName=AIRCRAFT_GROUP&VarDesc=AircraftGroup&VarType=Num&VarName=AIRCRAFT_TYPE&VarDesc=AircraftType&VarType=Char&VarName=AIRCRAFT_CONFIG&VarDesc=AircraftConfig&VarType=Num&VarName=YEAR&VarDesc=Year&VarType=Num&VarName=QUARTER&VarDesc=Quarter&VarType=Num&VarName=MONTH&VarDesc=Month&VarType=Num&VarName=DISTANCE_GROUP&VarDesc=DistanceGroup&VarType=Num&VarName=CLASS&VarDesc=Class&VarType=Char&VarName=DATA_SOURCE&VarDesc=DataSource&VarType=Char'
|
122
|
-
t = RemoteTable.new :url => url, :form_data => form_data, :compression => :zip, :glob => '/*.csv'
|
123
|
-
assert_equal 'United States of America', t.rows.first['DEST_COUNTRY_NAME']
|
124
|
-
end
|
125
|
-
|
126
157
|
should "take the last of values if the header is duplicated" do
|
127
158
|
t = RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tujrgUOwDSLWb-P4KCt1qBg')
|
128
159
|
assert_equal '2', t.rows.first['dup_header']
|
@@ -252,30 +283,6 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
252
283
|
assert_equal 99, t.rows[5]['MAP DATA population (millions) 2002'].to_i
|
253
284
|
end
|
254
285
|
|
255
|
-
should "open a CSV inside a zip file" do
|
256
|
-
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
|
257
|
-
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
258
|
-
assert_equal 'NSX', t.rows.first['carline name']
|
259
|
-
assert_equal 'TOYOTA', t.rows.last['Manufacturer']
|
260
|
-
assert_equal 'RAV4 SOFT TOP 4WD', t.rows.last['carline name']
|
261
|
-
end
|
262
|
-
|
263
|
-
should "open a fixed-width file with an inline schema inside a zip file" do
|
264
|
-
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
265
|
-
:filename => 'Gd6-dsc.txt',
|
266
|
-
:format => :fixed_width,
|
267
|
-
:crop => 21..26, # inclusive
|
268
|
-
:cut => '2-',
|
269
|
-
:select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
270
|
-
:schema => [[ 'code', 2, { :type => :string } ],
|
271
|
-
[ 'spacer', 2 ],
|
272
|
-
[ 'name', 52, { :type => :string } ]])
|
273
|
-
assert_equal 'regular grade gasoline (octane number of 87)', t.rows.first['name']
|
274
|
-
assert_equal 'R', t.rows.first['code']
|
275
|
-
assert_equal 'electricity', t.rows.last['name']
|
276
|
-
assert_equal 'El', t.rows.last['code']
|
277
|
-
end
|
278
|
-
|
279
286
|
should "open an XLS with a parser" do
|
280
287
|
ma_1990_01 = {"month"=>1, "cost"=>"54.0", "locatable"=>"Massachusetts (State)", "year"=>1990}
|
281
288
|
ga_1990_01 = {"month"=>1, "cost"=>"50.7", "locatable"=>"Georgia (State)", "year"=>1990}
|
@@ -287,17 +294,17 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
287
294
|
assert t.rows.include?(ga_1990_01)
|
288
295
|
end
|
289
296
|
|
290
|
-
should "provide a row_hash on demand" do
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
end
|
297
|
+
# should "provide a row_hash on demand" do
|
298
|
+
# t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
299
|
+
# :filename => 'Gd6-dsc.txt',
|
300
|
+
# :format => :fixed_width,
|
301
|
+
# :crop => 21..26, # inclusive
|
302
|
+
# :cut => '2-',
|
303
|
+
# :select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
304
|
+
# :schema => [[ 'code', 2, { :type => :string } ],
|
305
|
+
# [ 'spacer', 2 ],
|
306
|
+
# [ 'name', 52, { :type => :string } ]])
|
307
|
+
# assert_equal 'a8a5d7f17b56772723c657eb62b0f238', t.rows.first['row_hash']
|
308
|
+
# end
|
302
309
|
end
|
303
310
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 2
|
8
|
-
-
|
9
|
-
version: 0.2.
|
8
|
+
- 17
|
9
|
+
version: 0.2.17
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Seamus Abshere
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-04-
|
18
|
+
date: 2010-04-28 00:00:00 -04:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|