RubyGems - imw - Versions diffs - 0.2.7 → 0.2.8 - Mend

imw 0.2.7 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (93) hide show

data/Gemfile +23 -0
data/Gemfile.lock +47 -0
data/LICENSE +20 -674
data/README.rdoc +3 -4
data/VERSION +1 -1
data/lib/imw.rb +64 -35
data/lib/imw/dataset.rb +12 -2
data/lib/imw/formats.rb +4 -2
data/lib/imw/formats/delimited.rb +96 -36
data/lib/imw/formats/excel.rb +69 -101
data/lib/imw/formats/json.rb +3 -5
data/lib/imw/formats/pdf.rb +71 -0
data/lib/imw/formats/yaml.rb +3 -5
data/lib/imw/metadata.rb +66 -0
data/lib/imw/metadata/contains_metadata.rb +44 -0
data/lib/imw/metadata/dsl.rb +111 -0
data/lib/imw/metadata/field.rb +65 -0
data/lib/imw/metadata/schema.rb +227 -0
data/lib/imw/metadata/schematized.rb +27 -0
data/lib/imw/parsers.rb +1 -0
data/lib/imw/parsers/flat.rb +44 -0
data/lib/imw/resource.rb +36 -224
data/lib/imw/schemes.rb +3 -1
data/lib/imw/schemes/hdfs.rb +12 -1
data/lib/imw/schemes/http.rb +1 -2
data/lib/imw/schemes/local.rb +139 -16
data/lib/imw/schemes/remote.rb +14 -9
data/lib/imw/schemes/s3.rb +12 -0
data/lib/imw/schemes/sql.rb +117 -0
data/lib/imw/tools.rb +5 -3
data/lib/imw/tools/downloader.rb +63 -0
data/lib/imw/tools/summarizer.rb +21 -10
data/lib/imw/utils.rb +10 -0
data/lib/imw/utils/dynamically_extendable.rb +137 -0
data/lib/imw/utils/error.rb +3 -0
data/lib/imw/utils/extensions.rb +0 -4
data/lib/imw/utils/extensions/array.rb +6 -7
data/lib/imw/utils/extensions/hash.rb +3 -5
data/lib/imw/utils/extensions/string.rb +3 -3
data/lib/imw/utils/has_uri.rb +114 -0
data/spec/data/{sample.csv → formats/delimited/sample.csv} +1 -1
data/spec/data/{sample.tsv → formats/delimited/sample.tsv} +0 -0
data/spec/data/formats/delimited/with_schema/ace-hardware-locations.tsv +11 -0
data/spec/data/formats/delimited/with_schema/all-countries-ip-address-to-geolocation-data.tsv +16 -0
data/spec/data/formats/delimited/with_schema/complete-list-of-starbucks-locations.tsv +11 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-myspace-application-counts.tsv +12 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-latlong.tsv +13 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv +22 -0
data/spec/data/formats/delimited/with_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv +22 -0
data/spec/data/formats/delimited/without_schema/ace-hardware-locations.tsv +10 -0
data/spec/data/formats/delimited/without_schema/all-countries-ip-address-to-geolocation-data.tsv +15 -0
data/spec/data/formats/delimited/without_schema/complete-list-of-starbucks-locations.tsv +10 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-cumulative-word-count-from-from-dec.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-adds-by-zip-cod.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-myspace-application-counts.tsv +11 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-latlong.tsv +12 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-user-count-by-zip-code.tsv +21 -0
data/spec/data/formats/delimited/without_schema/myspace-user-activity-stream-word-count-by-day-from-december-200.tsv +21 -0
data/spec/data/formats/excel/sample.xls +0 -0
data/spec/data/formats/json/sample.json +1 -0
data/spec/data/formats/none/sample +650 -0
data/spec/data/formats/sgml/sample.xml +617 -0
data/spec/data/formats/text/sample.txt +650 -0
data/spec/data/formats/yaml/sample.yaml +410 -0
data/spec/data/schema-tabular.yaml +11 -0
data/spec/imw/formats/delimited_spec.rb +34 -2
data/spec/imw/formats/excel_spec.rb +55 -0
data/spec/imw/formats/json_spec.rb +3 -3
data/spec/imw/formats/sgml_spec.rb +4 -4
data/spec/imw/formats/yaml_spec.rb +3 -3
data/spec/imw/metadata/field_spec.rb +26 -0
data/spec/imw/metadata/schema_spec.rb +27 -0
data/spec/imw/metadata_spec.rb +39 -0
data/spec/imw/parsers/line_parser_spec.rb +1 -1
data/spec/imw/resource_spec.rb +0 -100
data/spec/imw/schemes/hdfs_spec.rb +19 -13
data/spec/imw/schemes/local_spec.rb +59 -3
data/spec/imw/schemes/s3_spec.rb +4 -0
data/spec/imw/utils/dynamically_extendable_spec.rb +69 -0
data/spec/imw/utils/has_uri_spec.rb +55 -0
data/spec/spec_helper.rb +1 -2
data/spec/support/random.rb +4 -4
metadata +58 -17
data/CHANGELOG +0 -0
data/TODO +0 -18
data/spec/data/sample.json +0 -782
data/spec/data/sample.txt +0 -131
data/spec/data/sample.xml +0 -653
data/spec/data/sample.yaml +0 -651
data/spec/spec.opts +0 -4
data/spec/support/extensions.rb +0 -18

data/spec/data/sample.yaml DELETED Viewed

@@ -1,651 +0,0 @@
----
-- id: 001
-  name: Gray-bellied Night Monkey
-  genus: Aotus
-  species: lemurinus
-- id: 002
-  name: Panamanian Night Monkey
-  genus: Aotus
-  species: zonalis
-- id: 003
-  name: Hernández-Camacho's Night Monkey
-  genus: Aotus
-  species: jorgehernandezi
-- id: 004
-  name: Gray-handed Night Monkey
-  genus: Aotus
-  species: griseimembra
-- id: 005
-  name: Hershkovitz's Night Monkey
-  genus: Aotus
-  species: hershkovitzi
-- id: 006
-  name: Brumback's Night Monkey
-  genus: Aotus
-  species: brumbacki
-- id: 007
-  name: Three-striped Night Monkey
-  genus: Aotus
-  species: trivirgatus
-- id: 008
-  name: Spix's Night Monkey
-  genus: Aotus
-  species: vociferans
-- id: 009
-  name: Malaysian Lar Gibbon
-  genus: Hylobates
-  species: lar lar
-- id: 010
-  name: Carpenter's Lar Gibbon
-  genus: Hylobates
-  species: lar carpenteri
-- id: 011
-  name: Central Lar Gibbon
-  genus: Hylobates
-  species: lar entelloides
-- id: 012
-  name: Sumatran Lar Gibbon
-  genus: Hylobates
-  species: lar vestitus
-- id: 013
-  name: Yunnan Lar Gibbon
-  genus: Hylobates
-  species: lar yunnanensis
-- id: 014
-  name: Mountain Agile Gibbon
-  genus: Hylobates
-  species: agilis agilis
-- id: 015
-  name: Bornean White-bearded Gibbon
-  genus: Hylobates
-  species: agilis albibarbis
-- id: 016
-  name: Lowland Agile Gibbon
-  genus: Hylobates
-  species: agilis unko
-- id: 017
-  name: Müller's Gray Gibbon
-  genus: Hylobates
-  species: muelleri muelleri
-- id: 018
-  name: Abbott's Gray Gibbon
-  genus: Hylobates
-  species: muelleri abbotti
-- id: 019
-  name: Northern Gray Gibbon
-  genus: Hylobates
-  species: muelleri funereus
-- id: 020
-  name: Black Tamarin
-  genus: Saguinas
-  species: niger
-- id: 021
-  name: Black-mantled Tamarin
-  genus: Saguinas
-  species: nigricollis
-- id: 022
-  name: Brown-mantled Tamarin
-  genus: Saguinas
-  species: fuscicollis
-- id: 023
-  name: Cottontop Tamarin or Pinché Tamarin
-  genus: Saguinas
-  species: oedipus
-- id: 024
-  name: Emperor Tamarin
-  genus: Saguinas
-  species: imperator
-- id: 025
-  name: Geoffroy's Tamarin
-  genus: Saguinas
-  species: geoffroyi
-- id: 026
-  name: Golden-mantled Tamarin
-  genus: Saguinas
-  species: tripartitus
-- id: 027
-  name: Graells's Tamarin
-  genus: Saguinas
-  species: graellsi
-- id: 028
-  name: Martins's Tamarin
-  genus: Saguinas
-  species: martinsi
-- id: 029
-  name: Mottle-faced Tamarin
-  genus: Saguinas
-  species: inustus
-- id: 030
-  name: Moustached Tamarin
-  genus: Saguinas
-  species: mystax
-- id: 031
-  name: Pied Tamarin
-  genus: Saguinas
-  species: bicolor
-- id: 032
-  name: Red-capped Tamarin
-  genus: Saguinas
-  species: pileatus
-- id: 033
-  name: Red-handed Tamarin
-  genus: Saguinas
-  species: midas
-- id: 034
-  name: White-footed Tamarin
-  genus: Saguinas
-  species: leucopus
-- id: 035
-  name: White-lipped Tamarin
-  genus: Saguinas
-  species: labiatus
-- id: 036
-  name: White-mantled Tamarin
-  genus: Saguinas
-  species: melanoleucus
-- id: 037
-  name: Allen's Swamp Monkey
-  genus: Allenopithecus
-  species: nigroviridis
-- id: 038
-  name: Angolan Talapoin
-  genus: Miopithecus
-  species: talapoin
-- id: 039
-  name: Gabon Talapoin
-  genus: Miopithecus
-  species: ogouensis
-- id: 040
-  name: Patas Monkey
-  genus: Erythrocebus
-  species: patas
-- id: 041
-  name: Green Monkey
-  genus: Chlorocebus
-  species: sabaeus
-- id: 042
-  name: Grivet
-  genus: Chlorocebus
-  species: aethiops
-- id: 043
-  name: Bale Mountains Vervet
-  genus: Chlorocebus
-  species: djamdjamensis
-- id: 044
-  name: Tantalus Monkey
-  genus: Chlorocebus
-  species: tantalus
-- id: 045
-  name: Vervet Monkey
-  genus: Chlorocebus
-  species: pygerythrus
-- id: 046
-  name: Malbrouck
-  genus: Chlorocebus
-  species: cynosuros
-- id: 047
-  name: Dryas Monkey or Salongo Monkey
-  genus: Cercopithecus
-  species: dryas
-- id: 048
-  name: Diana Monkey
-  genus: Cercopithecus
-  species: diana
-- id: 049
-  name: Roloway Monkey
-  genus: Cercopithecus
-  species: roloway
-- id: 050
-  name: Greater Spot-nosed Monkey
-  genus: Cercopithecus
-  species: nictitans
-- id: 051
-  name: Blue Monkey
-  genus: Cercopithecus
-  species: mitis
-- id: 052
-  name: Silver Monkey
-  genus: Cercopithecus
-  species: doggetti
-- id: 053
-  name: Golden Monkey
-  genus: Cercopithecus
-  species: kandti
-- id: 054
-  name: Sykes's Monkey
-  genus: Cercopithecus
-  species: albogularis
-- id: 055
-  name: Mona Monkey
-  genus: Cercopithecus
-  species: mona
-- id: 056
-  name: Campbell's Mona Monkey
-  genus: Cercopithecus
-  species: campbelli
-- id: 057
-  name: Lowe's Mona Monkey
-  genus: Cercopithecus
-  species: lowei
-- id: 058
-  name: Crested Mona Monkey
-  genus: Cercopithecus
-  species: pogonias
-- id: 059
-  name: Wolf's Mona Monkey
-  genus: Cercopithecus
-  species: wolfi
-- id: 060
-  name: Dent's Mona Monkey
-  genus: Cercopithecus
-  species: denti
-- id: 061
-  name: Lesser Spot-nosed Monkey
-  genus: Cercopithecus
-  species: petaurista
-- id: 062
-  name: White-throated Guenon
-  genus: Cercopithecus
-  species: erythrogaster
-- id: 063
-  name: Sclater's Guenon
-  genus: Cercopithecus
-  species: sclateri
-- id: 064
-  name: Red-eared Guenon
-  genus: Cercopithecus
-  species: erythrotis
-- id: 065
-  name: Moustached Guenon
-  genus: Cercopithecus
-  species: cephus
-- id: 066
-  name: Red-tailed Monkey
-  genus: Cercopithecus
-  species: ascanius
-- id: 067
-  name: L'Hoest's Monkey
-  genus: Cercopithecus
-  species: lhoesti
-- id: 068
-  name: Preuss's Monkey
-  genus: Cercopithecus
-  species: preussi
-- id: 069
-  name: Sun-tailed Monkey
-  genus: Cercopithecus
-  species: solatus
-- id: 070
-  name: Hamlyn's Monkey
-  genus: Cercopithecus
-  species: hamlyni
-- id: 071
-  name: De Brazza's Monkey
-  genus: Cercopithecus
-  species: neglectus
-- id: 072
-  name: Barbary Macaque
-  genus: Macaca
-  species: sylvanus
-- id: 073
-  name: Lion-tailed Macaque
-  genus: Macaca
-  species: silenus
-- id: 074
-  name: Southern Pig-tailed Macaque or Beruk
-  genus: Macaca
-  species: nemestrina
-- id: 075
-  name: Northern Pig-tailed Macaque
-  genus: Macaca
-  species: leonina
-- id: 076
-  name: Pagai Island Macaque or Bokkoi
-  genus: Macaca
-  species: pagensis
-- id: 077
-  name: Siberut Macaque
-  genus: Macaca
-  species: siberu
-- id: 078
-  name: Moor Macaque
-  genus: Macaca
-  species: maura
-- id: 079
-  name: Booted Macaque
-  genus: Macaca
-  species: ochreata
-- id: 080
-  name: Tonkean Macaque
-  genus: Macaca
-  species: tonkeana
-- id: 081
-  name: Heck's Macaque
-  genus: Macaca
-  species: hecki
-- id: 082
-  name: Gorontalo Macaque
-  genus: Macaca
-  species: nigrescens
-- id: 083
-  name: Celebes Crested Macaque or Black Ape
-  genus: Macaca
-  species: nigra
-- id: 084
-  name: Crab-eating Macaque or Long-tailed Macaque or Kera
-  genus: Macaca
-  species: fascicularis
-- id: 085
-  name: Stump-tailed Macaque or Bear Macaque
-  genus: Macaca
-  species: arctoides
-- id: 086
-  name: Rhesus Macaque
-  genus: Macaca
-  species: mulatta
-- id: 087
-  name: Formosan Rock Macaque
-  genus: Macaca
-  species: cyclopis
-- id: 088
-  name: Japanese Macaque
-  genus: Macaca
-  species: fuscata
-- id: 089
-  name: Toque Macaque
-  genus: Macaca
-  species: sinica
-- id: 090
-  name: Bonnet Macaque
-  genus: Macaca
-  species: radiata
-- id: 091
-  name: Assam Macaque
-  genus: Macaca
-  species: assamensis
-- id: 092
-  name: Tibetan Macaque or Milne-Edwards' Macaque
-  genus: Macaca
-  species: thibetana
-- id: 093
-  name: Arunachal Macaque or Munzala
-  genus: Macaca
-  species: munzala
-- id: 094
-  name: Grey-cheeked Mangabey
-  genus: Lophocebus
-  species: albigena
-- id: 095
-  name: Black Crested Mangabey
-  genus: Lophocebus
-  species: aterrimus
-- id: 096
-  name: Opdenbosch's Mangabey
-  genus: Lophocebus
-  species: opdenboschi
-- id: 097
-  name: Uganda Mangabey
-  genus: Lophocebus
-  species: ugandae
-- id: 098
-  name: Johnston's Mangabey
-  genus: Lophocebus
-  species: johnstoni
-- id: 099
-  name: Osman Hill's Mangabey
-  genus: Lophocebus
-  species: osmani
-- id: 100
-  name: Kipunji
-  genus: Rungwecebus
-  species: kipunji
-- id: 101
-  name: Hamadryas Baboon
-  genus: Papio
-  species: hamadryas
-- id: 102
-  name: Guinea Baboon
-  genus: Papio
-  species: papio
-- id: 103
-  name: Olive Baboon
-  genus: Papio
-  species: anubis
-- id: 104
-  name: Yellow Baboon
-  genus: Papio
-  species: cynocephalus
-- id: 105
-  name: Chacma Baboon
-  genus: Papio
-  species: ursinus
-- id: 106
-  name: Gelada
-  genus: Theropithecus
-  species: gelada
-- id: 107
-  name: Sooty Mangabey
-  genus: Cercocebus
-  species: atys
-- id: 108
-  name: Collared Mangabey
-  genus: Cercocebus
-  species: torquatus
-- id: 109
-  name: Agile Mangabey
-  genus: Cercocebus
-  species: agilis
-- id: 110
-  name: Golden-bellied Mangabey
-  genus: Cercocebus
-  species: chrysogaster
-- id: 111
-  name: Tana River Mangabey
-  genus: Cercocebus
-  species: galeritus
-- id: 112
-  name: Sanje Mangabey
-  genus: Cercocebus
-  species: sanjei
-- id: 113
-  name: Mandrill
-  genus: Mandrillus
-  species: sphinx
-- id: 114
-  name: Drill
-  genus: Mandrillus
-  species: leucophaeus
-- id: 115
-  name: Black Colobus
-  genus: Colobus
-  species: satanas
-- id: 116
-  name: Angola Colobus
-  genus: Colobus
-  species: angolensis
-- id: 117
-  name: King Colobus
-  genus: Colobus
-  species: polykomos
-- id: 118
-  name: Ursine Colobus
-  genus: Colobus
-  species: vellerosus
-- id: 119
-  name: Mantled Guereza
-  genus: Colobus
-  species: guereza
-- id: 120
-  name: Western Red Colobus
-  genus: Piliocolobus
-  species: badius
-- id: 121
-  name: Pennant's Colobus
-  genus: Piliocolobus
-  species: pennantii
-- id: 122
-  name: Preuss's Red Colobus
-  genus: Piliocolobus
-  species: preussi
-- id: 123
-  name: Thollon's Red Colobus
-  genus: Piliocolobus
-  species: tholloni
-- id: 124
-  name: Central African Red Colobus
-  genus: Piliocolobus
-  species: foai
-- id: 125
-  name: Ugandan Red Colobus
-  genus: Piliocolobus
-  species: tephrosceles
-- id: 126
-  name: Uzungwa Red Colobus
-  genus: Piliocolobus
-  species: gordonorum
-- id: 127
-  name: Zanzibar Red Colobus
-  genus: Piliocolobus
-  species: kirkii
-- id: 128
-  name: Tana River Red Colobus
-  genus: Piliocolobus
-  species: rufomitratus
-- id: 129
-  name: Olive Colobus
-  genus: Procolobus
-  species: verus
-- id: 130
-  name: Maroon Leaf Monkey
-  genus: Presbytis
-  species: rubicunda