remote_table 0.2.16 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/remote_table/file.rb +4 -4
- data/lib/remote_table/package.rb +2 -1
- data/lib/remote_table/request.rb +6 -8
- data/lib/remote_table.rb +20 -0
- data/remote_table.gemspec +2 -2
- data/test/remote_table_test.rb +65 -58
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.17
|
data/lib/remote_table/file.rb
CHANGED
@@ -59,13 +59,13 @@ class RemoteTable
|
|
59
59
|
|
60
60
|
def skip_rows!
|
61
61
|
return unless skip
|
62
|
-
|
62
|
+
RemoteTable.backtick_with_reporting "cat #{path} | tail -n +#{skip + 1} > #{path}.tmp"
|
63
63
|
FileUtils.mv "#{path}.tmp", path
|
64
64
|
end
|
65
65
|
|
66
66
|
def convert_file_to_utf8!
|
67
67
|
return if encoding == 'UTF-8' or encoding == 'UTF8'
|
68
|
-
|
68
|
+
RemoteTable.backtick_with_reporting "iconv -c -f #{encoding} -t UTF-8 #{path} > #{path}.tmp"
|
69
69
|
FileUtils.mv "#{path}.tmp", path
|
70
70
|
end
|
71
71
|
|
@@ -75,13 +75,13 @@ class RemoteTable
|
|
75
75
|
|
76
76
|
def cut_columns!
|
77
77
|
return unless cut
|
78
|
-
|
78
|
+
RemoteTable.backtick_with_reporting "cat #{path} | cut -c #{cut} > #{path}.tmp"
|
79
79
|
FileUtils.mv "#{path}.tmp", path
|
80
80
|
end
|
81
81
|
|
82
82
|
def crop_rows!
|
83
83
|
return unless crop
|
84
|
-
|
84
|
+
RemoteTable.backtick_with_reporting "cat #{path} | tail -n +#{crop.first} | head -n #{crop.last - crop.first + 1} > #{path}.tmp"
|
85
85
|
FileUtils.mv "#{path}.tmp", path
|
86
86
|
end
|
87
87
|
|
data/lib/remote_table/package.rb
CHANGED
@@ -47,7 +47,8 @@ class RemoteTable
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def move_and_process(path, extname, cmd, args)
|
50
|
-
|
50
|
+
FileUtils.mv path, "#{path}.#{extname}"
|
51
|
+
RemoteTable.backtick_with_reporting "#{cmd} #{path}.#{extname} #{args}"
|
51
52
|
end
|
52
53
|
|
53
54
|
# ex. A: 2007-01.csv.gz (compression not capable of storing multiple files)
|
data/lib/remote_table/request.rb
CHANGED
@@ -20,16 +20,14 @@ class RemoteTable
|
|
20
20
|
if parsed_url.scheme == 'file'
|
21
21
|
parsed_url.path
|
22
22
|
else
|
23
|
-
|
24
|
-
curl
|
25
|
-
--
|
26
|
-
--
|
27
|
-
--
|
28
|
-
|
29
|
-
"#{parsed_url}" \
|
23
|
+
RemoteTable.backtick_with_reporting %{
|
24
|
+
curl
|
25
|
+
--header "Expect: "
|
26
|
+
--location
|
27
|
+
#{"--data \"#{form_data}\"" if form_data.present?}
|
28
|
+
"#{parsed_url}"
|
30
29
|
--output "#{path}"
|
31
30
|
}
|
32
|
-
`#{cmd}`
|
33
31
|
path
|
34
32
|
end
|
35
33
|
end
|
data/lib/remote_table.rb
CHANGED
@@ -56,6 +56,26 @@ class RemoteTable
|
|
56
56
|
def <=>(other)
|
57
57
|
raise "Not implemented"
|
58
58
|
end
|
59
|
+
|
60
|
+
protected
|
61
|
+
|
62
|
+
# TODO this should probably live somewhere else
|
63
|
+
def self.backtick_with_reporting(cmd)
|
64
|
+
cmd = cmd.gsub /\s+/m, ' '
|
65
|
+
cmd = cmd + ' 2>&1' unless cmd.include? '2>'
|
66
|
+
output = `#{cmd}`
|
67
|
+
unless $?.success?
|
68
|
+
raise %{
|
69
|
+
From the remote_table gem...
|
70
|
+
|
71
|
+
Command failed:
|
72
|
+
#{cmd}
|
73
|
+
|
74
|
+
Output:
|
75
|
+
#{output}
|
76
|
+
}
|
77
|
+
end
|
78
|
+
end
|
59
79
|
|
60
80
|
private
|
61
81
|
|
data/remote_table.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{remote_table}
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.17"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
|
12
|
-
s.date = %q{2010-04-
|
12
|
+
s.date = %q{2010-04-28}
|
13
13
|
s.description = %q{Remotely open and parse Excel XLS, ODS, CSV and fixed-width tables.}
|
14
14
|
s.email = %q{seamus@abshere.net}
|
15
15
|
s.extra_rdoc_files = [
|
data/test/remote_table_test.rb
CHANGED
@@ -50,21 +50,7 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
50
50
|
]
|
51
51
|
end
|
52
52
|
|
53
|
-
if ENV['
|
54
|
-
end
|
55
|
-
|
56
|
-
if ENV['OLD'] == 'true'
|
57
|
-
should "read an HTML table made with frontpage" do
|
58
|
-
t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-E.htm",
|
59
|
-
:encoding => 'US-ASCII',
|
60
|
-
:row_xpath => '//table/tr[2]/td/table/tr',
|
61
|
-
:column_xpath => 'td'
|
62
|
-
assert_equal 'E110', t.rows.first['Designator']
|
63
|
-
assert_equal 'EMBRAER', t.rows.first['Manufacturer']
|
64
|
-
assert_equal 'EZKC', t.rows.last['Designator']
|
65
|
-
assert_equal 'EZ King Cobra', t.rows.last['Model']
|
66
|
-
end
|
67
|
-
|
53
|
+
if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
|
68
54
|
should "open an XLS inside a zip file" do
|
69
55
|
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/02data.zip', :filename => 'guide_jan28.xls')
|
70
56
|
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
@@ -79,6 +65,58 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
79
65
|
assert_equal nil, t.rows.first[:Manufacturer]
|
80
66
|
end
|
81
67
|
|
68
|
+
should "open a CSV inside a zip file" do
|
69
|
+
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
|
70
|
+
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
71
|
+
assert_equal 'NSX', t.rows.first['carline name']
|
72
|
+
assert_equal 'TOYOTA', t.rows.last['Manufacturer']
|
73
|
+
assert_equal 'RAV4 SOFT TOP 4WD', t.rows.last['carline name']
|
74
|
+
end
|
75
|
+
|
76
|
+
should "open a fixed-width file with an inline schema inside a zip file" do
|
77
|
+
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
78
|
+
:filename => 'Gd6-dsc.txt',
|
79
|
+
:format => :fixed_width,
|
80
|
+
:crop => 21..26, # inclusive
|
81
|
+
:cut => '2-',
|
82
|
+
:select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
83
|
+
:schema => [[ 'code', 2, { :type => :string } ],
|
84
|
+
[ 'spacer', 2 ],
|
85
|
+
[ 'name', 52, { :type => :string } ]])
|
86
|
+
assert_equal 'regular grade gasoline (octane number of 87)', t.rows.first['name']
|
87
|
+
assert_equal 'R', t.rows.first['code']
|
88
|
+
assert_equal 'electricity', t.rows.last['name']
|
89
|
+
assert_equal 'El', t.rows.last['code']
|
90
|
+
end
|
91
|
+
|
92
|
+
should "send form data, follow redirects and use a filename glob" do
|
93
|
+
url = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
|
94
|
+
form_data = 'UserTableName=T_100_Segment__All_Carriers&DBShortName=Air_Carriers&RawDataTable=T_T100_SEGMENT_ALL_CARRIER&sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D1+AND+YEAR%3D2008&varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&grouplist=&suml=&sumRegion=&filter1=title%3D&filter2=title%3D&geo=All%A0&time=January&timename=Month&GEOGRAPHY=All&XYEAR=2008&FREQUENCY=1&AllVars=All&VarName=DEPARTURES_SCHEDULED&VarDesc=DepScheduled&VarType=Num&VarName=DEPARTURES_PERFORMED&VarDesc=DepPerformed&VarType=Num&VarName=PAYLOAD&VarDesc=Payload&VarType=Num&VarName=SEATS&VarDesc=Seats&VarType=Num&VarName=PASSENGERS&VarDesc=Passengers&VarType=Num&VarName=FREIGHT&VarDesc=Freight&VarType=Num&VarName=MAIL&VarDesc=Mail&VarType=Num&VarName=DISTANCE&VarDesc=Distance&VarType=Num&VarName=RAMP_TO_RAMP&VarDesc=RampToRamp&VarType=Num&VarName=AIR_TIME&VarDesc=AirTime&VarType=Num&VarName=UNIQUE_CARRIER&VarDesc=UniqueCarrier&VarType=Char&VarName=AIRLINE_ID&VarDesc=AirlineID&VarType=Num&VarName=UNIQUE_CARRIER_NAME&VarDesc=UniqueCarrierName&VarType=Char&VarName=UNIQUE_CARRIER_ENTITY&VarDesc=UniqCarrierEntity&VarType=Char&VarName=REGION&VarDesc=CarrierRegion&VarType=Char&VarName=CARRIER&VarDesc=Carrier&VarType=Char&VarName=CARRIER_NAME&VarDesc=CarrierName&VarType=Char&VarName=CARRIER_GROUP&VarDesc=CarrierGroup&VarType=Num&VarName=CARRIER_GROUP_NEW&VarDesc=CarrierGroupNew&VarType=Num&VarName=ORIGIN&VarDesc=Origin&VarType=Char&VarName=ORIGIN_CITY_NAME&VarDesc=OriginCityName&VarType=Char&VarName=ORIGIN_CITY_NUM&VarDesc=OriginCityNum&VarType=Num&VarName=ORIGIN_STATE_ABR&VarDesc=OriginState&VarType=Char&VarName=ORIGIN_STATE_FIPS&VarDesc=OriginStateFips&VarType=Char&VarName=ORIGIN_STATE_NM&VarDesc=OriginStateName&VarType=Char&VarName=ORIGIN_COUNTRY&VarDesc=OriginCountry&VarType=Char&VarName=ORIGIN_COUNTRY_NAME&VarDesc=OriginCountryName&VarType=Char&VarName=ORIGIN_WAC&VarDesc=OriginWac&VarType=Num&VarName=DEST&VarDesc=Dest&VarType=Char&VarName=DEST_CITY_NAME&VarDesc=DestCityName&VarType=Char&VarName=DEST_CITY_NUM&VarDesc=DestCityNum&VarType=Num&VarName=DEST_STATE_ABR&VarDesc=DestState&VarType=Char&VarName=DEST_STATE_FIPS&VarDesc=DestStateFips&VarType=Char&VarName=DEST_STATE_NM&VarDesc=DestStateName&VarType=Char&VarName=DEST_COUNTRY&VarDesc=DestCountry&VarType=Char&VarName=DEST_COUNTRY_NAME&VarDesc=DestCountryName&VarType=Char&VarName=DEST_WAC&VarDesc=DestWac&VarType=Num&VarName=AIRCRAFT_GROUP&VarDesc=AircraftGroup&VarType=Num&VarName=AIRCRAFT_TYPE&VarDesc=AircraftType&VarType=Char&VarName=AIRCRAFT_CONFIG&VarDesc=AircraftConfig&VarType=Num&VarName=YEAR&VarDesc=Year&VarType=Num&VarName=QUARTER&VarDesc=Quarter&VarType=Num&VarName=MONTH&VarDesc=Month&VarType=Num&VarName=DISTANCE_GROUP&VarDesc=DistanceGroup&VarType=Num&VarName=CLASS&VarDesc=Class&VarType=Char&VarName=DATA_SOURCE&VarDesc=DataSource&VarType=Char'
|
95
|
+
t = RemoteTable.new :url => url, :form_data => form_data, :compression => :zip, :glob => '/*.csv'
|
96
|
+
assert_equal 'United States of America', t.rows.first['DEST_COUNTRY_NAME']
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
|
101
|
+
should "connect using HTTPS if available" do
|
102
|
+
t = RemoteTable.new(:url => 'https://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA')
|
103
|
+
assert_equal 'Gulf Coast', t.rows.first['PAD district name']
|
104
|
+
assert_equal 'AL', t.rows.first['State']
|
105
|
+
assert_equal 'Rocky Mountain', t.rows.last['PAD district name']
|
106
|
+
assert_equal 'WY', t.rows.last['State']
|
107
|
+
end
|
108
|
+
|
109
|
+
should "read an HTML table made with frontpage" do
|
110
|
+
t = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-E.htm",
|
111
|
+
:encoding => 'US-ASCII',
|
112
|
+
:row_xpath => '//table/tr[2]/td/table/tr',
|
113
|
+
:column_xpath => 'td'
|
114
|
+
assert_equal 'E110', t.rows.first['Designator']
|
115
|
+
assert_equal 'EMBRAER', t.rows.first['Manufacturer']
|
116
|
+
assert_equal 'EZKC', t.rows.last['Designator']
|
117
|
+
assert_equal 'EZ King Cobra', t.rows.last['Model']
|
118
|
+
end
|
119
|
+
|
82
120
|
should "hash rows without paying attention to order" do
|
83
121
|
x = ActiveSupport::OrderedHash.new
|
84
122
|
x[:a] = 1
|
@@ -116,13 +154,6 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
116
154
|
assert_equal 'Rocky Mountain', t.rows.last[4]
|
117
155
|
end
|
118
156
|
|
119
|
-
should "send form data, follow redirects and use a filename glob" do
|
120
|
-
url = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
|
121
|
-
form_data = 'UserTableName=T_100_Segment__All_Carriers&DBShortName=Air_Carriers&RawDataTable=T_T100_SEGMENT_ALL_CARRIER&sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D1+AND+YEAR%3D2008&varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&grouplist=&suml=&sumRegion=&filter1=title%3D&filter2=title%3D&geo=All%A0&time=January&timename=Month&GEOGRAPHY=All&XYEAR=2008&FREQUENCY=1&AllVars=All&VarName=DEPARTURES_SCHEDULED&VarDesc=DepScheduled&VarType=Num&VarName=DEPARTURES_PERFORMED&VarDesc=DepPerformed&VarType=Num&VarName=PAYLOAD&VarDesc=Payload&VarType=Num&VarName=SEATS&VarDesc=Seats&VarType=Num&VarName=PASSENGERS&VarDesc=Passengers&VarType=Num&VarName=FREIGHT&VarDesc=Freight&VarType=Num&VarName=MAIL&VarDesc=Mail&VarType=Num&VarName=DISTANCE&VarDesc=Distance&VarType=Num&VarName=RAMP_TO_RAMP&VarDesc=RampToRamp&VarType=Num&VarName=AIR_TIME&VarDesc=AirTime&VarType=Num&VarName=UNIQUE_CARRIER&VarDesc=UniqueCarrier&VarType=Char&VarName=AIRLINE_ID&VarDesc=AirlineID&VarType=Num&VarName=UNIQUE_CARRIER_NAME&VarDesc=UniqueCarrierName&VarType=Char&VarName=UNIQUE_CARRIER_ENTITY&VarDesc=UniqCarrierEntity&VarType=Char&VarName=REGION&VarDesc=CarrierRegion&VarType=Char&VarName=CARRIER&VarDesc=Carrier&VarType=Char&VarName=CARRIER_NAME&VarDesc=CarrierName&VarType=Char&VarName=CARRIER_GROUP&VarDesc=CarrierGroup&VarType=Num&VarName=CARRIER_GROUP_NEW&VarDesc=CarrierGroupNew&VarType=Num&VarName=ORIGIN&VarDesc=Origin&VarType=Char&VarName=ORIGIN_CITY_NAME&VarDesc=OriginCityName&VarType=Char&VarName=ORIGIN_CITY_NUM&VarDesc=OriginCityNum&VarType=Num&VarName=ORIGIN_STATE_ABR&VarDesc=OriginState&VarType=Char&VarName=ORIGIN_STATE_FIPS&VarDesc=OriginStateFips&VarType=Char&VarName=ORIGIN_STATE_NM&VarDesc=OriginStateName&VarType=Char&VarName=ORIGIN_COUNTRY&VarDesc=OriginCountry&VarType=Char&VarName=ORIGIN_COUNTRY_NAME&VarDesc=OriginCountryName&VarType=Char&VarName=ORIGIN_WAC&VarDesc=OriginWac&VarType=Num&VarName=DEST&VarDesc=Dest&VarType=Char&VarName=DEST_CITY_NAME&VarDesc=DestCityName&VarType=Char&VarName=DEST_CITY_NUM&VarDesc=DestCityNum&VarType=Num&VarName=DEST_STATE_ABR&VarDesc=DestState&VarType=Char&VarName=DEST_STATE_FIPS&VarDesc=DestStateFips&VarType=Char&VarName=DEST_STATE_NM&VarDesc=DestStateName&VarType=Char&VarName=DEST_COUNTRY&VarDesc=DestCountry&VarType=Char&VarName=DEST_COUNTRY_NAME&VarDesc=DestCountryName&VarType=Char&VarName=DEST_WAC&VarDesc=DestWac&VarType=Num&VarName=AIRCRAFT_GROUP&VarDesc=AircraftGroup&VarType=Num&VarName=AIRCRAFT_TYPE&VarDesc=AircraftType&VarType=Char&VarName=AIRCRAFT_CONFIG&VarDesc=AircraftConfig&VarType=Num&VarName=YEAR&VarDesc=Year&VarType=Num&VarName=QUARTER&VarDesc=Quarter&VarType=Num&VarName=MONTH&VarDesc=Month&VarType=Num&VarName=DISTANCE_GROUP&VarDesc=DistanceGroup&VarType=Num&VarName=CLASS&VarDesc=Class&VarType=Char&VarName=DATA_SOURCE&VarDesc=DataSource&VarType=Char'
|
122
|
-
t = RemoteTable.new :url => url, :form_data => form_data, :compression => :zip, :glob => '/*.csv'
|
123
|
-
assert_equal 'United States of America', t.rows.first['DEST_COUNTRY_NAME']
|
124
|
-
end
|
125
|
-
|
126
157
|
should "take the last of values if the header is duplicated" do
|
127
158
|
t = RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tujrgUOwDSLWb-P4KCt1qBg')
|
128
159
|
assert_equal '2', t.rows.first['dup_header']
|
@@ -252,30 +283,6 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
252
283
|
assert_equal 99, t.rows[5]['MAP DATA population (millions) 2002'].to_i
|
253
284
|
end
|
254
285
|
|
255
|
-
should "open a CSV inside a zip file" do
|
256
|
-
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv')
|
257
|
-
assert_equal 'ACURA', t.rows.first['Manufacturer']
|
258
|
-
assert_equal 'NSX', t.rows.first['carline name']
|
259
|
-
assert_equal 'TOYOTA', t.rows.last['Manufacturer']
|
260
|
-
assert_equal 'RAV4 SOFT TOP 4WD', t.rows.last['carline name']
|
261
|
-
end
|
262
|
-
|
263
|
-
should "open a fixed-width file with an inline schema inside a zip file" do
|
264
|
-
t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
265
|
-
:filename => 'Gd6-dsc.txt',
|
266
|
-
:format => :fixed_width,
|
267
|
-
:crop => 21..26, # inclusive
|
268
|
-
:cut => '2-',
|
269
|
-
:select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
270
|
-
:schema => [[ 'code', 2, { :type => :string } ],
|
271
|
-
[ 'spacer', 2 ],
|
272
|
-
[ 'name', 52, { :type => :string } ]])
|
273
|
-
assert_equal 'regular grade gasoline (octane number of 87)', t.rows.first['name']
|
274
|
-
assert_equal 'R', t.rows.first['code']
|
275
|
-
assert_equal 'electricity', t.rows.last['name']
|
276
|
-
assert_equal 'El', t.rows.last['code']
|
277
|
-
end
|
278
|
-
|
279
286
|
should "open an XLS with a parser" do
|
280
287
|
ma_1990_01 = {"month"=>1, "cost"=>"54.0", "locatable"=>"Massachusetts (State)", "year"=>1990}
|
281
288
|
ga_1990_01 = {"month"=>1, "cost"=>"50.7", "locatable"=>"Georgia (State)", "year"=>1990}
|
@@ -287,17 +294,17 @@ class RemoteTableTest < Test::Unit::TestCase
|
|
287
294
|
assert t.rows.include?(ga_1990_01)
|
288
295
|
end
|
289
296
|
|
290
|
-
should "provide a row_hash on demand" do
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
end
|
297
|
+
# should "provide a row_hash on demand" do
|
298
|
+
# t = RemoteTable.new(:url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
|
299
|
+
# :filename => 'Gd6-dsc.txt',
|
300
|
+
# :format => :fixed_width,
|
301
|
+
# :crop => 21..26, # inclusive
|
302
|
+
# :cut => '2-',
|
303
|
+
# :select => lambda { |row| /\A[A-Z]/.match row['code'] },
|
304
|
+
# :schema => [[ 'code', 2, { :type => :string } ],
|
305
|
+
# [ 'spacer', 2 ],
|
306
|
+
# [ 'name', 52, { :type => :string } ]])
|
307
|
+
# assert_equal 'a8a5d7f17b56772723c657eb62b0f238', t.rows.first['row_hash']
|
308
|
+
# end
|
302
309
|
end
|
303
310
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 2
|
8
|
-
-
|
9
|
-
version: 0.2.
|
8
|
+
- 17
|
9
|
+
version: 0.2.17
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Seamus Abshere
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-04-
|
18
|
+
date: 2010-04-28 00:00:00 -04:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|