mode 0.0.5 → 0.0.7

data/spec/fixtures/country-codes/datapackage.json

@@ -0,0 +1,142 @@
+{
+  "name": "country-codes",
+  "title": "Comprehensive country codes: ISO 3166, ITU, ISO 4217 currency codes and many more",
+  "format": "csv",
+  "datapackage_version": "1.0-beta.3",
+  "licenses": [
+    {
+      "id": "odc-pddl",
+      "name": "Public Domain Dedication and License",
+      "version": "1.0",
+      "url": "http://opendatacommons.org/licenses/pddl/1.0/"
+    }
+  ],
+  "sources": [
+    {
+      "name": "International Organization for Standardization",
+      "web": "http://www.iso.org/iso/country_codes/iso_3166_code_lists.htm"
+    },
+    {
+      "name": "SIX Interbank Clearing Ltd (on behalf of ISO)",
+      "web": "http://www.currency-iso.org/dam/downloads/dl_iso_table_a1.xls"
+    },
+    {
+      "name": "Statoids",
+      "web": "http://www.statoids.com/wab.html"
+    }
+  ],
+  "resources": [
+    {
+      "url": "https://raw.github.com/datasets/country-codes/master/data/country-codes.csv",
+      "path": "data/country-codes.csv",
+      "schema": {
+        "fields": [
+          {
+            "name": "name",
+            "description": "Country's official English short name",
+            "type": "string"
+          },
+          {
+            "name": "name_fr",
+            "description": "Country's offical French short name",
+            "type": "string"
+          },
+          {
+            "name": "ISO3166-1-Alpha-2",
+            "description": "Alpha-2 codes from ISO 3166-1",
+            "type": "string"
+          },
+          {
+            "name": "ISO3166-1-Alpha-3",
+            "description": "Alpha-3 codes from ISO 3166-1 (synonymous with World Bank Codes)",
+            "type": "string"
+          },
+          {
+            "name": "ISO3166-1-numeric",
+            "description": "Numeric codes from ISO 3166-1 (synonymous with UN Statistics M49 Codes)",
+            "type": "integer"
+          },
+          {
+            "name": "ITU",
+            "description": "Codes assigned by the International Telecommunications Union",
+            "type": "string"
+          },
+          {
+            "name": "MARC",
+            "description": "MAchine-Readable Cataloging codes from the Library of Congress",
+            "type": "string"
+          },
+          {
+            "name": "WMO",
+            "description": "Country abbreviations by the World Meteorological Organization",
+            "type": "string"
+          },
+          {
+            "name": "DS",
+            "description": "Distinguishing signs of vehicles in international traffic",
+            "type": "string"
+          },
+          {
+            "name": "Dial",
+            "description": "Country code from ITU-T recommendation E.164, sometimes followed by area code",
+            "type": "string"
+          },
+          {
+            "name": "FIFA",
+            "description": "Codes assigned by the Fédération Internationale de Football Association",
+            "type": "string"
+          },
+          {
+            "name": "FIPS",
+            "description": "Codes from the U.S. standard FIPS PUB 10-4",
+            "type": "string"
+          },
+          {
+            "name": "GAUL",
+            "description": "Global Administrative Unit Layers from the Food and Agriculture Organization",
+            "type": "integer"
+          },
+          {
+            "name": "IOC",
+            "description": "Codes assigned by the International Olympics Committee",
+            "type": "string"
+          },
+          {
+            "name": "currency_alphabetic_code",
+            "description": "ISO 4217 currency alphabetic code",
+            "type": "string"
+          },
+          {
+            "name": "currency_country_name",
+            "description": "ISO 4217 country name",
+            "type": "string"
+          },
+          {
+            "name": "currency_minor_unit",
+            "description": "ISO 4217 currency number of minor units",
+            "type": "integer"
+          },
+          {
+            "name": "currency_name",
+            "description": "ISO 4217 currency name",
+            "type": "string"
+          },
+          {
+            "name": "currency_numeric_code",
+            "description": "ISO 4217 currency numeric code",
+            "type": "integer"
+          },
+          {
+            "name": "is_independent",
+            "description": "Country status, based on the CIA World Factbook",
+            "type": "string"
+          }
+        ]
+      }
+    }
+  ],
+  "maintainers":[{
+    "name": "Evan Wheeler",
+    "web": "https://github.com/datasets/country-codes"
+  }]
+}

data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py

@@ -0,0 +1,370 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim: ai ts=4 sts=4 et sw=4
+import codecs
+import urllib
+import argparse
+import json
+import operator
+import collections
+
+from lxml import html
+from lxml import etree
+
+# some ANSI colors, etc
+BLUE = '\033[94m'
+GREEN = '\033[92m'
+YELLOW = '\033[93m'
+RED = '\033[91m'
+BOLD = '\033[1m'
+ENDC = '\033[0m'
+
+
+def print_info(string):
+    print GREEN + string + ENDC
+
+
+def print_warn(string):
+    print YELLOW + string + ENDC
+
+
+def print_error(string):
+    print RED + string + ENDC
+
+
+def process_statoids_row(tr):
+    row = []
+    for td in tr.iterchildren():
+        if len(td.keys()) > 0:
+            if td.get('colspan') is not None:
+                # if a cell is taking up more than one column,
+                # append the same number of blanks to the row
+                assert td.get('colspan').isdigit()
+                for col in xrange(int(td.get('colspan'))):
+                    row.append('')
+                continue
+        if len(td.getchildren()) == 1:
+            if td.find('.//br') is not None:
+                if len(td.getchildren()) == 1:
+                    if td.getchildren()[0].tag == 'br':
+                        td.text = td.text + " " + td.getchildren()[0].tail
+                        row.append(td.text)
+                        continue
+            if td.find("code") is not None:
+                # some cells contain more than one code,
+                # so append a list also containing the code
+                # that appears after the child element (<br>)
+                if len(td.find("code").getchildren()) > 0:
+                    if td.find('.//br') is not None:
+                        row.append(td.find('code').text + ',' + td.find('.//br').tail)
+                        continue
+                    if td.find('.//a') is not None:
+                        anchor = td.find('.//a')
+                        # UK has 4 FIFA codes
+                        if row[1] == "GB":
+                            assert anchor.text == "1"
+                            row.append("ENG,NIR,SCO,WAL")
+                            continue
+                        # MARC treats United States Minor Outlying Islands
+                        # as five countries
+                        if row[1] == "UM":
+                            assert anchor.text == "b"
+                            row.append("ji,xf,wk,uc,up")
+                            continue
+                # some cells contain anchor to footnote,
+                # so append only the content of the code element
+                row.append(td.find("code").text)
+                continue
+            else:
+                if td.find('.//a') is not None:
+                    anchor = td.find('.//a')
+                    # FIPS treats United States Minor Outlying Islands
+                    # as nine countries
+                    if len(row) > 1 and row[1] == "UM":
+                        assert anchor.text == "a"
+                        row.append("FQ,HQ,DQ,JQ,KQ,MQ,BQ,LQ,WQ")
+                        continue
+        row.append(td.text_content())
+    return row
+
+def clean_line(line):
+    try:
+        line = line.decode('utf8')
+        line = line.rstrip()
+        if ';' in line:
+            semi = line.index(';')
+            name = line[:semi]
+            alpha2 = line[semi + 1:]
+            return (name, alpha2)
+        return (None, None)
+    except UnicodeDecodeError:
+        print_warn('Unable to decode country name: %s' % line)
+
+def capitalize_country_name(name):
+    # replace all-caps name with capitalized country name
+    cap_list = []
+    always_lower = ['AND', 'THE', 'OF', 'PART', 'DA', 'DE', 'ET', 'DU', 'DES',
+                    'LA']
+    for w in name.split():
+        if w == 'MCDONALD':
+            cap_list.append('McDonald')
+        if w.find('.') > 0:
+            cap_list.append(w.upper())
+            continue
+        if w.find('\'') > 0:
+            # d'Ivoire instead of D'ivoire
+            s = w.split('\'')
+            if len(s[0]) == 1:
+                cap_list.append(s[0].lower() + '\'' + s[1].capitalize())
+                continue
+        if w.find('-') > 0:
+            # Timor-Leste instead of Timor-leste
+            cap_list.append('-'.join([s.capitalize() for s in w.split('-')]))
+            continue
+
+        if w.startswith('('):
+            w = w.replace('(', '')
+            if w in always_lower:
+                w = w.lower()
+            else:
+                w = w.capitalize()
+            cap_list.append('(' + w)
+            continue
+
+        if w[-1] == ')':
+            w = w.replace(')', '')
+            if w in always_lower:
+                w = w.lower()
+            else:
+                w = w.capitalize()
+            cap_list.append(w + ')')
+            continue
+
+        if w in always_lower:
+            cap_list.append(w.lower())
+            continue
+        cap_list.append(w.capitalize())
+
+    capitalized = " ".join(cap_list)
+    return capitalized
+
+
+def get_currency_data(country_info, en_names):
+    # fetch iso currency codes
+    currency_url = "http://www.currency-iso.org/dam/downloads/table_a1.xml"
+    print_info('Fetching currency codes...')
+    currencies_xml_str = urllib.urlopen(currency_url).read()
+    currencies = etree.fromstring(currencies_xml_str)
+
+    # map source's tag names to our property names
+    currency_tag_map = {
+        u"CtryNm": u"currency_country_name",
+        u"CcyNm": u"currency_name",
+        u"Ccy": u"currency_alphabetic_code",
+        u"CcyNbr": u"currency_numeric_code",
+        u"CcyMnrUnts": u"currency_minor_unit",
+        u"AddtlInf": u"currency_additional_info"
+    }
+    # reconcile country names, add entries for non-country-based currencies
+    currency_country_name_map = {
+        u"MACEDONIA, THE FORMER \nYUGOSLAV REPUBLIC OF": "MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF",
+        u"SAINT HELENA, ASCENSION AND \nTRISTAN DA CUNHA": "SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA",
+        u"CONGO, THE DEMOCRATIC REPUBLIC OF": "CONGO, THE DEMOCRATIC REPUBLIC OF THE",
+        u"HEARD ISLAND AND McDONALD ISLANDS": "HEARD ISLAND AND MCDONALD ISLANDS",
+        u"KOREA, DEMOCRATIC PEOPLE’S REPUBLIC OF": "KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF",
+        u"LAO PEOPLE’S DEMOCRATIC REPUBLIC": "LAO PEOPLE'S DEMOCRATIC REPUBLIC",
+        u"SERBIA ": "SERBIA",
+        u"PALESTINIAN TERRITORY, OCCUPIED": "PALESTINE, STATE OF",
+        u"Vatican City State (HOLY SEE)": "HOLY SEE (VATICAN CITY STATE)",
+        u"VIRGIN ISLANDS (BRITISH)": "VIRGIN ISLANDS, BRITISH",
+        u"VIRGIN ISLANDS (US)": "VIRGIN ISLANDS, U.S.",
+        u"MEMBER COUNTRIES OF THE AFRICAN DEVELOPMENT BANK GROUP": None,
+        u"INTERNATIONAL MONETARY FUND (IMF)": None,
+        u"SISTEMA UNITARIO DE COMPENSACION REGIONAL DE PAGOS \"SUCRE\"": None,
+        u"EUROPEAN UNION": None,
+        u"ZZ01_Bond Markets Unit European_EURCO": None,
+        u"ZZ02_Bond Markets Unit European_EMU-6": None,
+        u"ZZ03_Bond Markets Unit European_EUA-9": None,
+        u"ZZ04_Bond Markets Unit European_EUA-17": None,
+        u"ZZ05_UIC-Franc": None,
+        u"ZZ06_Testing_Code": None,
+        u"ZZ07_No_Currency": None,
+        u"ZZ08_Gold": None,
+        u"ZZ09_Palladium": None,
+        u"ZZ10_Platinum": None,
+        u"ZZ11_Silver": None,
+    }
+
+    def process_element(country):
+        currency_dict = {}
+        for currency_tag in country.iterchildren():
+            # ignore newly added additional info field
+            if currency_tag_map[currency_tag.tag] == "currency_additional_info":
+                break
+            # skip 'same day', 'next day', etc variations
+            elif (currency_tag_map[currency_tag.tag] == "currency_name") and (len(currency_tag.items()) > 0):
+                if currency_tag.items()[0][0] == 'IsFund':
+                    break
+            else:
+                currency_dict.update({
+                    currency_tag_map[currency_tag.tag]: currency_tag.text})
+                currency_alpha2 = None
+                # remove random line breaks, etc
+                currency_name = currency_dict['currency_country_name'].replace(u'\xa0', u'').replace(u'\n', u'').replace(u'\r', u'')
+                if currency_name is not None:
+                    # replace name with line breaks, etc removed
+                    currency_dict['currency_country_name'] = currency_name
+                try:
+                    currency_alpha2 = en_names[currency_name]
+                except KeyError:
+                    currency_alpha2 = en_names.get(
+                        currency_country_name_map.get(currency_name))
+
+                if currency_alpha2:
+                    country_info[currency_alpha2].update(currency_dict)
+                else:
+                    if currency_name not in currency_country_name_map:
+                        print_warn('Failed to match currency data for country: "%s"'
+                                   % currency_name)
+        return
+
+    for iso_currency_table in currencies.iterchildren():
+        for country in iso_currency_table.iterchildren():
+            process_element(country)
+
+    return country_info
+
+def fetch_and_write(options):
+    # fetch ISO short names in English and French
+    print_info('Fetching English country names and codes...')
+    iso_names_en = urllib.urlretrieve('http://www.iso.org/iso/list-en1-semic-3.txt')
+    print_info('Fetching French country names and codes...')
+    iso_names_fr = urllib.urlretrieve('http://www.iso.org/iso/list-fr1-semic.txt')
+
+    # dict for combining en and fr names
+    # {alpha2: {'name': en, 'name_fr': fr}}
+    iso_names = {}
+
+    # dict for looking up alpha2 from name
+    en_names = {}
+
+    # urllib.urlretrieve returns a tuple of (localfile, headers)
+    with open(iso_names_en[0], "rU") as fin:
+        for line in fin:
+            name, alpha2 = clean_line(line)
+            if name and alpha2:
+                iso_names.update({alpha2: {'name': name}})
+                en_names.update({name: alpha2})
+
+    with open(iso_names_fr[0], "rU") as fin:
+        for line in fin:
+            name, alpha2 = clean_line(line)
+            if name and alpha2:
+                if alpha2 in iso_names:
+                    # alpha2 should be in iso_names because
+                    # english was parsed first,
+                    # so append french name to list
+                    names = iso_names[alpha2]
+                    names.update({'name_fr': name})
+                    iso_names.update({alpha2: names})
+                else:
+                    # hopefully this doesnt happen, but
+                    # in case there was no english name,
+                    # add french with a blank space where
+                    # english should be
+                    names = {'name': '', 'name_fr': name}
+                    iso_names.update({alpha2: names})
+
+    # fetch content of statoids.com country code page
+    statoids_url = "http://www.statoids.com/wab.html"
+    print_info('Fetching other country codes...')
+    content = urllib.urlopen(statoids_url).read()
+    doc = html.fromstring(content)
+
+    # i dislike some of statoid's column names, so here i have renamed
+    # a few to be more descriptive
+    column_names = ["Entity", "ISO3166-1-Alpha-2", "ISO3166-1-Alpha-3",
+                    "ISO3166-1-numeric", "ITU", "FIPS", "IOC", "FIFA", "DS",
+                    "WMO", "GAUL", "MARC", "Dial", "is_independent"]
+    alpha2_key = "ISO3166-1-Alpha-2"
+
+    # comment out the preceding two lines and
+    # uncomment these lines to use statoids.com column names
+    """
+    column_names = []
+    alpha2_key = 'A-2'
+    for tr in doc.find_class('hd'):
+        for th in tr.iterchildren():
+            column_names.append(th.text_content())
+    """
+
+    # dict to hold dicts of all table rows
+    table_rows = {}
+
+    # the country code info is in a table where the trs have
+    # alternating classes of `e` and `o`
+    # so fetch half of the rows and zip each row together
+    # with the corresponding column name
+    for tr in doc.find_class('e'):
+        row = process_statoids_row(tr)
+        row_dict = collections.OrderedDict(zip(column_names, row))
+        # statoids-assigned 'Entity' name is not really a standard
+        row_dict.pop('Entity')
+        table_rows.update({row_dict[alpha2_key]: row_dict})
+
+    # and again for the other half
+    for tr in doc.find_class('o'):
+        row = process_statoids_row(tr)
+        row_dict = collections.OrderedDict(zip(column_names, row))
+        # statoids-assigned 'Entity' name is not really a standard
+        row_dict.pop('Entity')
+        table_rows.update({row_dict[alpha2_key]: row_dict})
+
+    # dict to hold combined country info
+    country_info = {}
+    keyed_by = options.key
+
+    # iterate through all the table_rows
+    # TODO this assumes that statoids will have all of
+    # the items that are pulled from iso.org
+    for alpha2, info in table_rows.iteritems():
+        # ignore this crap that was parsed from other tables on the page
+        if alpha2 in ['', 'Codes', 'Codes Codes', 'Codes Codes Codes']:
+            continue
+        cinfo = info
+        # add iso.org's names to combined dict of this country's info
+        cinfo.update(iso_names[alpha2])
+        # replace all-caps name with capitalized country name
+        cinfo.update({'name': capitalize_country_name(cinfo['name'])})
+        cinfo.update({'name_fr': capitalize_country_name(cinfo['name_fr'])})
+        # add combined dict to global (pun intented) data structure
+        ckey = cinfo[keyed_by]
+        country_info.update({ckey: cinfo})
+
+    country_info = get_currency_data(country_info, en_names)
+
+    # reorganize data for export
+    if options.as_list:
+        # if exporting as list, sort by country name
+        country_info = sorted(country_info.values(), key=operator.itemgetter('name'))
+    # dump dict as json to file
+    output_filename = "data/country-codes.json"
+    if options.outfile:
+        output_filename = options.outfile
+    f = open(output_filename, mode='w')
+    stream = codecs.getwriter('utf8')(f)
+    json.dump(country_info, stream, ensure_ascii=False, indent=2, encoding='utf-8')
+    print_info('Saved country data to: %s' % output_filename)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Fetch current ISO 3166 country codes and other standards and output as JSON file')
+    parser.add_argument("-o", "--output", dest="outfile", default="data/country-codes.json",
+                        help="write data to OUTFILE", metavar="OUTFILE")
+    parser.add_argument("-l", "--list", dest="as_list", default=False, action="store_true",
+                        help="export objects as a list of objects")
+    parser.add_argument("-k", "--key", dest="key", default="ISO3166-1-Alpha-2",
+                        help="export objects as a dict of objects keyed by KEY", metavar="KEY")
+
+    args = parser.parse_args()
+
+    fetch_and_write(args)