libis-format 0.9.32 → 0.9.33

data/tools/fido/pronomutils.py

@@ -1,200 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-PRONOM UTILS.
-
-PYTHON FUNCTION TO QUERY PRONOM VERSION
-AND DOWNLOAD SIGNATUREFILE
-USES PRONOM SOAP SERVICE
-
-Open Planets Foundation (http://www.openplanetsfoundation.org)
-See License.txt for license information.
-Download from: http://github.com/openplanets/fido/downloads
-Author: Maurice de Rooij (OPF/NANETH), 2012
-
-PRONOM UTILS is a library used by FIDO.
-FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions.
-PRONOM is available from http://www.nationalarchives.gov.uk/pronom/
-"""
-
-from __future__ import absolute_import
-
-import os
-import re
-import sys
-from xml.etree import ElementTree as ET
-from xml.etree.ElementTree import parse, ParseError
-from xml.parsers.expat import ExpatError, ParserCreate
-
-import six
-from six.moves import http_client
-
-from . import __version__, CONFIG_DIR
-
-
-def check_well_formedness(filename, error=False):
-    """
-    Check if a given file contains valid XML.
-
-    :param filename: file from which the XML is read.
-    :param error: whether or not print to `stderr` upon error.
-    :returns: whether the file contains valid XML.
-    """
-    parser = ParserCreate()
-    try:
-        parser.ParseFile(open(filename, "r"))
-    except ExpatError as e:
-        if error is not False:
-            sys.stderr.write("check_well_formedness: %s: %s;\n" % (filename, e))
-        return False
-    return True
-
-
-def get_pronom_signature(type_):
-    """
-    Get PRONOM signature.
-
-    Return latest signature file version number as int when `type_` equals
-    "version" or return latest signature XML file as string when `type_` equals
-    "file". Upon error, write to `stderr` and returls `False`.
-    """
-    try:
-        soapVersionContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileVersionV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
-        soapFileContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
-        soapVersionHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileVersionV1In\""""
-        soapFileHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileV1In\""""
-        if type_ == "version":
-            soapAction = soapVersionHeader
-            soapStr = soapVersionContainer
-        elif type_ == "file":
-            soapAction = soapFileHeader
-            soapStr = soapFileContainer
-        else:
-            sys.stderr.write("get_pronom_signature(): unknown type: " + type_)
-            return False
-        webservice = http_client.HTTP("www.nationalarchives.gov.uk")
-        webservice.putrequest("POST", "/pronom/service.asmx")
-        webservice.putheader("Host", "www.nationalarchives.gov.uk")
-        webservice.putheader("User-Agent", "PRONOM UTILS v{0} (OPF)".format(__version__))
-        webservice.putheader("Content-type", "text/xml; charset=\"UTF-8\"")
-        webservice.putheader("Content-length", "%d" % len(soapStr))
-        webservice.putheader("SOAPAction", soapAction)
-        try:
-            webservice.endheaders()
-        except Exception as e:
-            sys.stderr.write("get_pronom_signature(): failed to contact PRONOM;\n%s\n" % (e))
-            sys.exit()
-        webservice.send(soapStr)
-        statuscode, statusmessage, header = webservice.getreply()
-        if statuscode == 200:
-            xml = webservice.getfile()
-            if type_ == "version":
-                exp = re.compile("\<Version\>([0-9]{1,4})\<\/Version\>")
-                sigxml = exp.search(xml.read())
-                if len(sigxml.group(1)) > 0:
-                    return int(sigxml.group(1))
-                else:
-                    sys.stderr.write("get_pronom_signature(): could not parse VERSION from SOAP response: " + type_)
-                    return False
-            if type_ == "file":
-                exp = re.compile("\<SignatureFile\>.*\<\/SignatureFile\>")
-                sigxml = exp.search(xml.read())
-                sigtxt = sigxml.group(0) if sigxml else ''
-                if len(sigtxt) > 0:
-                    tmpfile = "./tmp_getPronomSignature.xml"
-                    with open(tmpfile, 'wb') as file_:
-                        file_.write("""<?xml version="1.0" encoding="UTF-8"?>""" + "\n")
-                        file_.write(sigtxt)
-                    if not check_well_formedness(tmpfile):
-                        os.unlink(tmpfile)
-                        sys.stderr.write("get_pronom_signature(): signaturefile not well formed")
-                        return False
-                    else:
-                        os.unlink(tmpfile)
-                        return """<?xml version="1.0" encoding="UTF-8"?>""" + "\n" + sigtxt
-                else:
-                    sys.stderr.write("get_pronom_signature(): could not parse XML from SOAP response: " + type_)
-                    return False
-        else:
-            sys.stderr.write("get_pronom_signature(): webservice error: '" + str(statuscode) + " " + statusmessage + "'\n")
-            return False
-        sys.stderr.write("get_pronom_signature(): unexpected return")
-        return False
-    except Exception as e:
-        sys.stderr.write("get_pronom_signature(): unknown error: " + str(e))
-        return False
-
-
-class LocalPronomVersions(object):
-    """
-    Parse local PRONOM signature versions XML file.
-
-    This is how the XML document should look like:
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <versions>
-        <pronomVersion>84</pronomVersion>
-        <pronomSignature>formats-v84.xml</pronomSignature>
-        <pronomContainerSignature>container-signature-20160121.xml</pronomContainerSignature>
-        <fidoExtensionSignature>format_extensions.xml</fidoExtensionSignature>
-        <updateScript>1.2.2</updateScript>
-    </versions>
-    """
-
-    PROPS_MAPPING = {
-        'pronom_version': 'pronomVersion',
-        'pronom_signature': 'pronomSignature',
-        'pronom_container_signature': 'pronomContainerSignature',
-        'fido_extension_signature': 'fidoExtensionSignature',
-        'update_script': 'updateScript',
-    }
-
-    ROOT_ELEMENT = 'versions'
-
-    def __init__(self, versions_file):
-        """Instantiate class based on the file indicated in `versions_file`."""
-        self.versions_file = versions_file
-        self.conf_dir = os.path.abspath(os.path.dirname(versions_file))
-        try:
-            self.tree = parse(versions_file)
-            self.root = self.tree.getroot()
-        except (ParseError, IOError):
-            self.root = ET.Element(self.ROOT_ELEMENT)
-            self.tree = ET.ElementTree(self.root)
-
-    def __getattr__(self, name):
-        """Extract the element's text content."""
-        if name in self.PROPS_MAPPING:
-            return self.root.find(self.PROPS_MAPPING[name]).text
-
-    def __setattr__(self, name, value):
-        """Update the element's text content."""
-        if name in self.PROPS_MAPPING:
-            try:
-                self.root.find(self.PROPS_MAPPING[name]).text = value
-            except AttributeError:
-                elem = ET.SubElement(self.root, self.PROPS_MAPPING[name])
-                elem.text = value
-        else:
-            object.__setattr__(self, name, value)
-
-    def get_zip_file(self):
-        """Obtain location to the PRONOM XML Zip file based on the current PRONOM version."""
-        return os.path.join(self.conf_dir, 'pronom-xml-v{}.zip'.format(self.pronom_version))
-
-    def get_signature_file(self):
-        """Obtain location to the current PRONOM signature file."""
-        return os.path.join(self.conf_dir, self.pronom_signature)
-
-    def write(self):
-        """Update versions.xml."""
-        # Check that all the fields are defined
-        for key, value in six.iteritems(self.PROPS_MAPPING):
-            if self.root.find(value) is None:
-                raise ValueError('Field {} has not been defined!'.format(key))
-        self.tree.write(self.versions_file, xml_declaration=True, method='xml', encoding='utf-8')
-
-
-def get_local_pronom_versions(config_dir=CONFIG_DIR):
-    """Return an instance of LocalPronomVersions loaded with `conf/versions.xml`."""
-    return LocalPronomVersions(os.path.join(config_dir, 'versions.xml'))

data/tools/fido/toxml.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-FIDO CSV output to XML.
-
-Author: Maurice de Rooij <maurice.de.rooij@nationaalarchief.nl>, September 2011
-
-Usage in combination with FIDO:
-- Windows: python fido.py [ARGS] | python toxml.py > output.xml
-- Linux: fido.py [ARGS] | toxml.py > output.xml
-
-Usage afterwards:
-- Windows: type output.csv | toxml.py > output.xml
-- Linux: cat output.csv | toxml.py > output.xml
-
-For difference in usage, see:
-- http://bugs.python.org/issue9390
-- http://support.microsoft.com/default.aspx?kbid=321788
-"""
-
-from __future__ import absolute_import
-
-import csv
-import sys
-
-from . import __version__
-from .pronomutils import get_local_pronom_versions
-
-
-def main():
-    """Generate XML as read from CSV and send it to the standard output stream."""
-    sys.stdout.write("""<?xml version="1.0" encoding="utf-8"?>
-<fido_output>
-    <versions>
-        <fido_version>{0}</fido_version>
-        <signature_version>{1}</signature_version>
-    </versions>""".format(__version__, get_local_pronom_versions().pronom_version))
-
-    reader = csv.reader(sys.stdin)
-
-    for row in reader:
-        sys.stdout.write("""
-    <file>
-        <filename>{0}</filename>
-        <status>{1}</status>
-        <matchtype>{2}</matchtype>
-        <time>{3}</time>
-        <puid>{4}</puid>
-        <mimetype>{5}</mimetype>
-        <formatname>{6}</formatname>
-        <signaturename>{7}</signaturename>
-        <filesize>{8}</filesize>
-    </file>""".format(row[6], row[0], row[8], row[1], row[2], row[7], row[3], row[4], row[5]))
-
-    sys.stdout.write("\n</fido_output>\n")
-
-
-if __name__ == '__main__':
-    main()

data/tools/fido/update_signatures.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-FIDO SIGNATURE UPDATER.
-
-Open Planets Foundation (http://www.openplanetsfoundation.org)
-See License.txt for license information.
-Download from: https://github.com/openplanets/fido/releases
-Author: Maurice de Rooij (NANETH), 2012
-
-FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions.
-PRONOM is available from http://www.nationalarchives.gov.uk/pronom/.
-"""
-
-from __future__ import print_function
-
-from argparse import ArgumentParser
-import os
-from shutil import rmtree
-import sys
-import time
-from xml.etree import ElementTree as CET
-import zipfile
-
-from six.moves.urllib.request import urlopen
-
-from . import __version__, CONFIG_DIR, query_yes_no
-from .prepare import run as prepare_pronom_to_fido
-from .pronomutils import check_well_formedness, get_local_pronom_versions, get_pronom_signature
-
-
-defaults = {
-    'signatureFileName': 'DROID_SignatureFile-v{0}.xml',
-    'pronomZipFileName': 'pronom-xml-v{0}.zip',
-    'fidoSignatureVersion': 'format_extensions.xml',
-    'containerVersion': 'container-signature-20160121.xml',  # container version is frozen and needs human attention before updating,
-}
-
-options = {
-    'http_throttle': 0.5,  # in secs, to prevent DoS of PRONOM server
-    'tmp_dir': os.path.join(CONFIG_DIR, 'tmp'),
-    'deleteTempDirectory': True,
-}
-
-
-def run(defaults=defaults):
-    """
-    Update PRONOM signatures.
-
-    Interactive script, requires keyboard input.
-    """
-    print("FIDO signature updater v{}".format(__version__))
-
-    try:
-        print("Contacting PRONOM...")
-        currentVersion = get_pronom_signature("version")
-        if not currentVersion:
-            sys.exit('Failed to obtain PRONOM signature file version number, please try again.')
-
-        print("Querying latest signaturefile version...")
-        signatureFile = os.path.join(CONFIG_DIR, defaults['signatureFileName'].format(currentVersion))
-        if os.path.isfile(signatureFile):
-            print("You already have the latest PRONOM signature file, version", currentVersion)
-            if not query_yes_no("Update anyway?"):
-                sys.exit('Aborting update...')
-
-        print("Downloading signature file version {}...".format(currentVersion))
-        currentFile = get_pronom_signature("file")
-        if not currentFile:
-            sys.exit('Failed to obtain PRONOM signature file, please try again.')
-        print("Writing {0}...".format(defaults['signatureFileName'].format(currentVersion)))
-        with open(signatureFile, 'wb') as file_:
-            file_.write(currentFile)
-
-        print("Extracting PRONOM PUID's from signature file...")
-        tree = CET.parse(signatureFile)
-        puids = []
-        for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"):
-            puids.append(node.get("PUID"))
-        numberPuids = len(puids)
-        print("Found {} PRONOM PUID's".format(numberPuids))
-
-        print("Downloading signatures can take a while")
-        if not query_yes_no("Continue and download signatures?"):
-            sys.exit('Aborting update...')
-        tmpdir = defaults['tmp_dir']
-        if os.path.isdir(tmpdir):
-            print("Found previously created temporary folder for download:", tmpdir)
-            resume_download = query_yes_no('Do you want to resume download (yes) or start over (no)?')
-            if resume_download:
-                print("Resuming download...")
-        else:
-            print("Creating temporary folder for download:", tmpdir)
-            try:
-                os.mkdir(tmpdir)
-            except:
-                pass
-        if not os.path.isdir(tmpdir):
-            print("Failed to create temporary folder for PUID's, using", tmpdir)
-
-        print("Downloading signatures, one moment please...")
-        one_percent = (float(numberPuids) / 100)
-        numfiles = 0
-        for puid in puids:
-            puidType, puidNum = puid.split("/")
-            puidFileName = "puid." + puidType + "." + puidNum + ".xml"
-            filename = os.path.join(tmpdir, puidFileName)
-            if os.path.isfile(filename) and check_well_formedness(filename) and resume_download:
-                numfiles += 1
-                continue
-            puid_url = "http://www.nationalarchives.gov.uk/pronom/{}.xml".format(puid)
-            try:
-                filehandle = urlopen(puid_url)
-            except Exception as e:
-                print("Failed to download signature file:", puid_url)
-                print("Error:", str(e))
-                sys.exit('Please restart and resume download.')
-            with open(filename, 'wb') as file_:
-                for lines in filehandle.readlines():
-                    file_.write(lines)
-            filehandle.close()
-            if not check_well_formedness(filename):
-                os.unlink(filename)
-                continue
-            numfiles += 1
-            percent = int(float(numfiles) / one_percent)
-            print(r"{}/{} files [{}%]".format(numfiles, numberPuids, percent))
-            time.sleep(defaults['http_throttle'])
-        print("100%")
-
-        print("Creating PRONOM zip...")
-        compression = zipfile.ZIP_DEFLATED if 'zlib' in sys.modules else zipfile.ZIP_STORED
-        modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'}
-        zf = zipfile.ZipFile(os.path.join(CONFIG_DIR, defaults['pronomZipFileName'].format(currentVersion)), mode='w')
-        print("Adding files with compression mode", modes[compression])
-        for puid in puids:
-            puidType, puidNum = puid.split("/")
-            puidFileName = "puid.{}.{}.xml".format(puidType, puidNum)
-            filename = os.path.join(tmpdir, puidFileName)
-            if os.path.isfile(filename):
-                zf.write(filename, arcname=puidFileName, compress_type=compression)
-                if defaults['deleteTempDirectory']:
-                    os.unlink(filename)
-        zf.close()
-
-        if defaults['deleteTempDirectory']:
-            print("Deleting temporary folder and files...")
-            rmtree(tmpdir, ignore_errors=True)
-
-        print('Updating versions.xml...')
-        versions = get_local_pronom_versions()
-        versions.pronom_version = str(currentVersion)
-        versions.pronom_signature = "formats-v" + str(currentVersion) + ".xml"
-        versions.pronom_container_signature = defaults['containerVersion']
-        versions.fido_extension_signature = defaults['fidoSignatureVersion']
-        versions.update_script = __version__
-        versions.write()
-
-        # TODO: there should be a check here to handle prepare.main exit() signal (-1/0/1/...)
-        print("Preparing to convert PRONOM formats to FIDO signatures...")
-        prepare_pronom_to_fido()
-        print("FIDO signatures successfully updated")
-
-    except KeyboardInterrupt:
-        sys.exit('Aborting update...')
-
-
-def main():
-    """Main CLI entrypoint."""
-    parser = ArgumentParser(description='Download and convert the latest PRONOM signatures')
-    parser.add_argument('-tmpdir', default=options['tmp_dir'], help='Location to store temporary files', dest='tmp_dir')
-    parser.add_argument('-keep_tmp', default=options['deleteTempDirectory'], help='Do not delete temporary files after completion', dest='deleteTempDirectory', action='store_false')
-    parser.add_argument('-http_throttle', default=options['http_throttle'], help='Time (in seconds) to wait between downloads', type=float, dest='http_throttle')
-    args = parser.parse_args()
-    opts = defaults.copy()
-    opts.update(vars(args))
-
-    run(opts)
-
-
-if __name__ == '__main__':
-    main()