RubyGems - imw - Versions diffs - 0.2.7 → 0.2.8 - Mend

imw 0.2.7 → 0.2.8

Files changed (93) hide show

data/Gemfile +23 -0
data/Gemfile.lock +47 -0
data/LICENSE +20 -674
data/README.rdoc +3 -4
data/VERSION +1 -1
data/lib/imw.rb +64 -35
data/lib/imw/dataset.rb +12 -2
data/lib/imw/formats.rb +4 -2
data/lib/imw/formats/delimited.rb +96 -36
data/lib/imw/formats/excel.rb +69 -101
data/lib/imw/formats/json.rb +3 -5
data/lib/imw/formats/pdf.rb +71 -0
data/lib/imw/formats/yaml.rb +3 -5
data/lib/imw/metadata.rb +66 -0
data/lib/imw/metadata/contains_metadata.rb +44 -0
data/lib/imw/metadata/dsl.rb +111 -0
data/lib/imw/metadata/field.rb +65 -0
data/lib/imw/metadata/schema.rb +227 -0
data/lib/imw/metadata/schematized.rb +27 -0
data/lib/imw/parsers.rb +1 -0
data/lib/imw/parsers/flat.rb +44 -0
data/lib/imw/resource.rb +36 -224
data/lib/imw/schemes.rb +3 -1
data/lib/imw/schemes/hdfs.rb +12 -1
data/lib/imw/schemes/http.rb +1 -2
data/lib/imw/schemes/local.rb +139 -16
data/lib/imw/schemes/remote.rb +14 -9
data/lib/imw/schemes/s3.rb +12 -0
data/lib/imw/schemes/sql.rb +117 -0
data/lib/imw/tools.rb +5 -3
data/lib/imw/tools/downloader.rb +63 -0
data/lib/imw/tools/summarizer.rb +21 -10
data/lib/imw/utils.rb +10 -0
data/lib/imw/utils/dynamically_extendable.rb +137 -0
data/lib/imw/utils/error.rb +3 -0
data/lib/imw/utils/extensions.rb +0 -4
data/lib/imw/utils/extensions/array.rb +6 -7
data/lib/imw/utils/extensions/hash.rb +3 -5
data/lib/imw/utils/extensions/string.rb +3 -3
data/lib/imw/utils/has_uri.rb +114 -0
data/spec/data/{sample.csv → formats/delimited/sample.csv} +1 -1
data/spec/data/{sample.tsv → formats/delimited/sample.tsv} +0 -0
data/spec/data/formats/delimited/with_schema/ace-hardware-locations.tsv +11 -0
data/spec/data/formats/delimited/with_schema/all-countries-ip-address-to-geolocation-data.tsv +16 -0
data/spec/data/formats/delimited/with_schema/complete-list-of-starbucks-locations.tsv +11 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-counts.tsv +12 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-latlong.tsv +13 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv +22 -0
data/spec/data/formats/delimited/without_schema/ace-hardware-locations.tsv +10 -0
data/spec/data/formats/delimited/without_schema/all-countries-ip-address-to-geolocation-data.tsv +15 -0
data/spec/data/formats/delimited/without_schema/complete-list-of-starbucks-locations.tsv +10 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-counts.tsv +11 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-latlong.tsv +12 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv +21 -0
data/spec/data/formats/excel/sample.xls +0 -0
data/spec/data/formats/json/sample.json +1 -0
data/spec/data/formats/none/sample +650 -0
data/spec/data/formats/sgml/sample.xml +617 -0
data/spec/data/formats/text/sample.txt +650 -0
data/spec/data/formats/yaml/sample.yaml +410 -0
data/spec/data/schema-tabular.yaml +11 -0
data/spec/imw/formats/delimited_spec.rb +34 -2
data/spec/imw/formats/excel_spec.rb +55 -0
data/spec/imw/formats/json_spec.rb +3 -3
data/spec/imw/formats/sgml_spec.rb +4 -4
data/spec/imw/formats/yaml_spec.rb +3 -3
data/spec/imw/metadata/field_spec.rb +26 -0
data/spec/imw/metadata/schema_spec.rb +27 -0
data/spec/imw/metadata_spec.rb +39 -0
data/spec/imw/parsers/line_parser_spec.rb +1 -1
data/spec/imw/resource_spec.rb +0 -100
data/spec/imw/schemes/hdfs_spec.rb +19 -13
data/spec/imw/schemes/local_spec.rb +59 -3
data/spec/imw/schemes/s3_spec.rb +4 -0
data/spec/imw/utils/dynamically_extendable_spec.rb +69 -0
data/spec/imw/utils/has_uri_spec.rb +55 -0
data/spec/spec_helper.rb +1 -2
data/spec/support/random.rb +4 -4
metadata +58 -17
data/CHANGELOG +0 -0
data/TODO +0 -18
data/spec/data/sample.json +0 -782
data/spec/data/sample.txt +0 -131
data/spec/data/sample.xml +0 -653
data/spec/data/sample.yaml +0 -651
data/spec/spec.opts +0 -4
data/spec/support/extensions.rb +0 -18

data/spec/data/sample.yaml DELETED Viewed

@@ -1,651 +0,0 @@
----
-- id: 001
-  name: Gray-bellied Night Monkey
-  genus: Aotus
-  species: lemurinus
-- id: 002
-  name: Panamanian Night Monkey
-  genus: Aotus
-  species: zonalis
-- id: 003
-  name: Hernández-Camacho's Night Monkey
-  genus: Aotus
-  species: jorgehernandezi
-- id: 004
-  name: Gray-handed Night Monkey
-  genus: Aotus
-  species: griseimembra
-- id: 005
-  name: Hershkovitz's Night Monkey
-  genus: Aotus
-  species: hershkovitzi
-- id: 006
-  name: Brumback's Night Monkey
-  genus: Aotus
-  species: brumbacki
-- id: 007
-  name: Three-striped Night Monkey
-  genus: Aotus
-  species: trivirgatus
-- id: 008
-  name: Spix's Night Monkey
-  genus: Aotus
-  species: vociferans
-- id: 009
-  name: Malaysian Lar Gibbon
-  genus: Hylobates
-  species: lar lar
-- id: 010
-  name: Carpenter's Lar Gibbon
-  genus: Hylobates
-  species: lar carpenteri
-- id: 011
-  name: Central Lar Gibbon
-  genus: Hylobates
-  species: lar entelloides
-- id: 012
-  name: Sumatran Lar Gibbon
-  genus: Hylobates
-  species: lar vestitus
-- id: 013
-  name: Yunnan Lar Gibbon
-  genus: Hylobates
-  species: lar yunnanensis
-- id: 014
-  name: Mountain Agile Gibbon
-  genus: Hylobates
-  species: agilis agilis
-- id: 015
-  name: Bornean White-bearded Gibbon
-  genus: Hylobates
-  species: agilis albibarbis
-- id: 016
-  name: Lowland Agile Gibbon
-  genus: Hylobates
-  species: agilis unko
-- id: 017
-  name: Müller's Gray Gibbon
-  genus: Hylobates
-  species: muelleri muelleri
-- id: 018
-  name: Abbott's Gray Gibbon
-  genus: Hylobates
-  species: muelleri abbotti
-- id: 019
-  name: Northern Gray Gibbon
-  genus: Hylobates
-  species: muelleri funereus
-- id: 020
-  name: Black Tamarin
-  genus: Saguinas
-  species: niger
-- id: 021
-  name: Black-mantled Tamarin
-  genus: Saguinas
-  species: nigricollis
-- id: 022
-  name: Brown-mantled Tamarin
-  genus: Saguinas
-  species: fuscicollis
-- id: 023
-  name: Cottontop Tamarin or Pinché Tamarin
-  genus: Saguinas
-  species: oedipus
-- id: 024
-  name: Emperor Tamarin
-  genus: Saguinas
-  species: imperator
-- id: 025
-  name: Geoffroy's Tamarin
-  genus: Saguinas
-  species: geoffroyi
-- id: 026
-  name: Golden-mantled Tamarin
-  genus: Saguinas
-  species: tripartitus
-- id: 027
-  name: Graells's Tamarin
-  genus: Saguinas
-  species: graellsi
-- id: 028
-  name: Martins's Tamarin
-  genus: Saguinas
-  species: martinsi
-- id: 029
-  name: Mottle-faced Tamarin
-  genus: Saguinas
-  species: inustus
-- id: 030
-  name: Moustached Tamarin
-  genus: Saguinas
-  species: mystax
-- id: 031
-  name: Pied Tamarin
-  genus: Saguinas
-  species: bicolor
-- id: 032
-  name: Red-capped Tamarin
-  genus: Saguinas
-  species: pileatus
-- id: 033
-  name: Red-handed Tamarin
-  genus: Saguinas
-  species: midas
-- id: 034
-  name: White-footed Tamarin
-  genus: Saguinas
-  species: leucopus
-- id: 035
-  name: White-lipped Tamarin
-  genus: Saguinas
-  species: labiatus
-- id: 036
-  name: White-mantled Tamarin
-  genus: Saguinas
-  species: melanoleucus
-- id: 037
-  name: Allen's Swamp Monkey
-  genus: Allenopithecus
-  species: nigroviridis
-- id: 038
-  name: Angolan Talapoin
-  genus: Miopithecus
-  species: talapoin
-- id: 039
-  name: Gabon Talapoin
-  genus: Miopithecus
-  species: ogouensis
-- id: 040
-  name: Patas Monkey
-  genus: Erythrocebus
-  species: patas
-- id: 041
-  name: Green Monkey
-  genus: Chlorocebus
-  species: sabaeus
-- id: 042
-  name: Grivet
-  genus: Chlorocebus
-  species: aethiops
-- id: 043
-  name: Bale Mountains Vervet
-  genus: Chlorocebus
-  species: djamdjamensis
-- id: 044
-  name: Tantalus Monkey
-  genus: Chlorocebus
-  species: tantalus
-- id: 045
-  name: Vervet Monkey
-  genus: Chlorocebus
-  species: pygerythrus
-- id: 046
-  name: Malbrouck
-  genus: Chlorocebus
-  species: cynosuros
-- id: 047
-  name: Dryas Monkey or Salongo Monkey
-  genus: Cercopithecus
-  species: dryas
-- id: 048
-  name: Diana Monkey
-  genus: Cercopithecus
-  species: diana
-- id: 049
-  name: Roloway Monkey
-  genus: Cercopithecus
-  species: roloway
-- id: 050
-  name: Greater Spot-nosed Monkey
-  genus: Cercopithecus
-  species: nictitans
-- id: 051
-  name: Blue Monkey
-  genus: Cercopithecus
-  species: mitis
-- id: 052
-  name: Silver Monkey
-  genus: Cercopithecus
-  species: doggetti
-- id: 053
-  name: Golden Monkey
-  genus: Cercopithecus
-  species: kandti
-- id: 054
-  name: Sykes's Monkey
-  genus: Cercopithecus
-  species: albogularis
-- id: 055
-  name: Mona Monkey
-  genus: Cercopithecus
-  species: mona
-- id: 056
-  name: Campbell's Mona Monkey
-  genus: Cercopithecus
-  species: campbelli
-- id: 057
-  name: Lowe's Mona Monkey
-  genus: Cercopithecus
-  species: lowei
-- id: 058
-  name: Crested Mona Monkey
-  genus: Cercopithecus
-  species: pogonias
-- id: 059
-  name: Wolf's Mona Monkey
-  genus: Cercopithecus
-  species: wolfi
-- id: 060
-  name: Dent's Mona Monkey
-  genus: Cercopithecus
-  species: denti
-- id: 061
-  name: Lesser Spot-nosed Monkey
-  genus: Cercopithecus
-  species: petaurista
-- id: 062
-  name: White-throated Guenon
-  genus: Cercopithecus
-  species: erythrogaster
-- id: 063
-  name: Sclater's Guenon
-  genus: Cercopithecus
-  species: sclateri
-- id: 064
-  name: Red-eared Guenon
-  genus: Cercopithecus
-  species: erythrotis
-- id: 065
-  name: Moustached Guenon
-  genus: Cercopithecus
-  species: cephus
-- id: 066
-  name: Red-tailed Monkey
-  genus: Cercopithecus
-  species: ascanius
-- id: 067
-  name: L'Hoest's Monkey
-  genus: Cercopithecus
-  species: lhoesti
-- id: 068
-  name: Preuss's Monkey
-  genus: Cercopithecus
-  species: preussi
-- id: 069
-  name: Sun-tailed Monkey
-  genus: Cercopithecus
-  species: solatus
-- id: 070
-  name: Hamlyn's Monkey
-  genus: Cercopithecus
-  species: hamlyni
-- id: 071
-  name: De Brazza's Monkey
-  genus: Cercopithecus
-  species: neglectus
-- id: 072
-  name: Barbary Macaque
-  genus: Macaca
-  species: sylvanus
-- id: 073
-  name: Lion-tailed Macaque
-  genus: Macaca
-  species: silenus
-- id: 074
-  name: Southern Pig-tailed Macaque or Beruk
-  genus: Macaca
-  species: nemestrina
-- id: 075
-  name: Northern Pig-tailed Macaque
-  genus: Macaca
-  species: leonina
-- id: 076
-  name: Pagai Island Macaque or Bokkoi
-  genus: Macaca
-  species: pagensis
-- id: 077
-  name: Siberut Macaque
-  genus: Macaca
-  species: siberu
-- id: 078
-  name: Moor Macaque
-  genus: Macaca
-  species: maura
-- id: 079
-  name: Booted Macaque
-  genus: Macaca
-  species: ochreata
-- id: 080
-  name: Tonkean Macaque
-  genus: Macaca
-  species: tonkeana
-- id: 081
-  name: Heck's Macaque
-  genus: Macaca
-  species: hecki
-- id: 082
-  name: Gorontalo Macaque
-  genus: Macaca
-  species: nigrescens
-- id: 083
-  name: Celebes Crested Macaque or Black Ape
-  genus: Macaca
-  species: nigra
-- id: 084
-  name: Crab-eating Macaque or Long-tailed Macaque or Kera
-  genus: Macaca
-  species: fascicularis
-- id: 085
-  name: Stump-tailed Macaque or Bear Macaque
-  genus: Macaca
-  species: arctoides
-- id: 086
-  name: Rhesus Macaque
-  genus: Macaca
-  species: mulatta
-- id: 087
-  name: Formosan Rock Macaque
-  genus: Macaca
-  species: cyclopis
-- id: 088
-  name: Japanese Macaque
-  genus: Macaca
-  species: fuscata
-- id: 089
-  name: Toque Macaque
-  genus: Macaca
-  species: sinica
-- id: 090
-  name: Bonnet Macaque
-  genus: Macaca
-  species: radiata
-- id: 091
-  name: Assam Macaque
-  genus: Macaca
-  species: assamensis
-- id: 092
-  name: Tibetan Macaque or Milne-Edwards' Macaque
-  genus: Macaca
-  species: thibetana
-- id: 093
-  name: Arunachal Macaque or Munzala
-  genus: Macaca
-  species: munzala
-- id: 094
-  name: Grey-cheeked Mangabey
-  genus: Lophocebus
-  species: albigena
-- id: 095
-  name: Black Crested Mangabey
-  genus: Lophocebus
-  species: aterrimus
-- id: 096
-  name: Opdenbosch's Mangabey
-  genus: Lophocebus
-  species: opdenboschi
-- id: 097
-  name: Uganda Mangabey
-  genus: Lophocebus
-  species: ugandae
-- id: 098
-  name: Johnston's Mangabey
-  genus: Lophocebus
-  species: johnstoni
-- id: 099
-  name: Osman Hill's Mangabey
-  genus: Lophocebus
-  species: osmani
-- id: 100
-  name: Kipunji
-  genus: Rungwecebus
-  species: kipunji
-- id: 101
-  name: Hamadryas Baboon
-  genus: Papio
-  species: hamadryas
-- id: 102
-  name: Guinea Baboon
-  genus: Papio
-  species: papio
-- id: 103
-  name: Olive Baboon
-  genus: Papio
-  species: anubis
-- id: 104
-  name: Yellow Baboon
-  genus: Papio
-  species: cynocephalus
-- id: 105
-  name: Chacma Baboon
-  genus: Papio
-  species: ursinus
-- id: 106
-  name: Gelada
-  genus: Theropithecus
-  species: gelada
-- id: 107
-  name: Sooty Mangabey
-  genus: Cercocebus
-  species: atys
-- id: 108
-  name: Collared Mangabey
-  genus: Cercocebus
-  species: torquatus
-- id: 109
-  name: Agile Mangabey
-  genus: Cercocebus
-  species: agilis
-- id: 110
-  name: Golden-bellied Mangabey
-  genus: Cercocebus
-  species: chrysogaster
-- id: 111
-  name: Tana River Mangabey
-  genus: Cercocebus
-  species: galeritus
-- id: 112
-  name: Sanje Mangabey
-  genus: Cercocebus
-  species: sanjei
-- id: 113
-  name: Mandrill
-  genus: Mandrillus
-  species: sphinx
-- id: 114
-  name: Drill
-  genus: Mandrillus
-  species: leucophaeus
-- id: 115
-  name: Black Colobus
-  genus: Colobus
-  species: satanas
-- id: 116
-  name: Angola Colobus
-  genus: Colobus
-  species: angolensis
-- id: 117
-  name: King Colobus
-  genus: Colobus
-  species: polykomos
-- id: 118
-  name: Ursine Colobus
-  genus: Colobus
-  species: vellerosus
-- id: 119
-  name: Mantled Guereza
-  genus: Colobus
-  species: guereza
-- id: 120
-  name: Western Red Colobus
-  genus: Piliocolobus
-  species: badius
-- id: 121
-  name: Pennant's Colobus
-  genus: Piliocolobus
-  species: pennantii
-- id: 122
-  name: Preuss's Red Colobus
-  genus: Piliocolobus
-  species: preussi
-- id: 123
-  name: Thollon's Red Colobus
-  genus: Piliocolobus
-  species: tholloni
-- id: 124
-  name: Central African Red Colobus
-  genus: Piliocolobus
-  species: foai
-- id: 125
-  name: Ugandan Red Colobus
-  genus: Piliocolobus
-  species: tephrosceles
-- id: 126
-  name: Uzungwa Red Colobus
-  genus: Piliocolobus
-  species: gordonorum
-- id: 127
-  name: Zanzibar Red Colobus
-  genus: Piliocolobus
-  species: kirkii
-- id: 128
-  name: Tana River Red Colobus
-  genus: Piliocolobus
-  species: rufomitratus
-- id: 129
-  name: Olive Colobus
-  genus: Procolobus
-  species: verus
-- id: 130
-  name: Maroon Leaf Monkey
-  genus: Presbytis
-  species: rubicunda