libis-format 0.9.5-java

data/tools/fido/pronomutils.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# PRONOM UTILS  
+#
+# PYTHON FUNCTION TO QUERY PRONOM VERSION 
+# AND DOWNLOAD SIGNATUREFILE
+# USES PRONOM SOAP SERVICE
+#
+# Open Planets Foundation (http://www.openplanetsfoundation.org)
+# See License.txt for license information.
+# Download from: http://github.com/openplanets/fido/downloads
+# Author: Maurice de Rooij (OPF/NANETH), 2012
+# 
+# PRONOM UTILS is a library used by FIDO 
+# FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions.
+# PRONOM is available from http://www.nationalarchives.gov.uk/pronom/
+#
+import sys
+from xml.dom import minidom
+__pronomutils__ = {'version' : '1.0.1'}
+
+def checkWellFormedness(filename,error=False):
+    """
+        usage: checkWellFormedness(filename)
+        arguments:
+        "filename": returns true if filename is a valid XML file
+        "error": whether or not print to stderr upon error
+    """
+    import xml.parsers.expat
+    parser = xml.parsers.expat.ParserCreate()
+    try:
+        parser.ParseFile(open(filename, "r"))
+    except Exception, e:
+        if error is not False:
+            sys.stderr.write("checkWellFormedness: %s: %s;\n" % (filename, e))
+        return False
+    return True
+
+def getPronomSignature(type):
+    """
+        usage: getPronomSignature(version|file)
+        arguments:
+        "version": returns latest signature file version number as int
+        "file": returns latest signature XML file as string
+        upon error: writes to stderr and returns false
+    """
+    try:
+        import httplib
+        import re
+        import os
+        soapVersionContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileVersionV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
+        soapFileContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
+        soapVersionHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileVersionV1In\""""
+        soapFileHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileV1In\""""
+        if type == "version":
+            soapAction = soapVersionHeader
+            soapStr = soapVersionContainer
+        elif type == "file":
+            soapAction = soapFileHeader
+            soapStr = soapFileContainer
+        else:
+            sys.stderr.write("getPronomSignature(): unknown type: "+type)
+            return False
+        webservice = httplib.HTTP("apps.nationalarchives.gov.uk")
+        webservice.putrequest("POST", "/pronom/service.asmx")
+        webservice.putheader("Host", "www.nationalarchives.gov.uk")
+        webservice.putheader("User-Agent", "PRONOM UTILS v{0} (OPF)".format(__pronomutils__['version']))
+        webservice.putheader("Content-type", "text/xml; charset=\"UTF-8\"")
+        webservice.putheader("Content-length", "%d" % len(soapStr))
+        webservice.putheader("SOAPAction", soapAction)
+        try:
+            webservice.endheaders()
+        except Exception, e:
+            sys.stderr.write("getPronomSignature(): failed to contact PRONOM;\n%s\n" % (e))
+            sys.exit()
+        webservice.send(soapStr)
+        statuscode, statusmessage, header = webservice.getreply()
+        if statuscode == 200:
+            xml = webservice.getfile()
+            if type == "version":
+                exp = re.compile("\<Version\>([0-9]{1,4})\<\/Version\>")
+                sigxml = exp.search(xml.read())
+                if len(sigxml.group(1)) > 0:
+                    return int(sigxml.group(1))
+                else:
+                    sys.stderr.write("getPronomSignature(): could not parse VERSION from SOAP response: "+type)
+                    return False
+            if type == "file":
+                exp = re.compile("\<SignatureFile\>.*\<\/SignatureFile\>")
+                sigxml = exp.search(xml.read())
+                sigtxt = sigxml.group(0) if sigxml else ''
+                if len(sigtxt) > 0:
+                    tmpfile = "./tmp_getPronomSignature.xml"
+                    tmp = open(tmpfile,'wb')
+                    tmp.write("""<?xml version="1.0" encoding="UTF-8"?>"""+"\n")
+                    tmp.write(sigtxt)
+                    tmp.close()
+                    if not checkWellFormedness(tmpfile):
+                        os.unlink(tmpfile)
+                        sys.stderr.write("getPronomSignature(): signaturefile not well formed")
+                        return False
+                    else:
+                        os.unlink(tmpfile)
+                        return """<?xml version="1.0" encoding="UTF-8"?>"""+"\n"+sigtxt
+                else:
+                    sys.stderr.write("getPronomSignature(): could not parse XML from SOAP response: "+type)
+                    return False
+        else:
+            sys.stderr.write("getPronomSignature(): webservice error: '"+str(statuscode)+" "+statusmessage+"'\n")
+            return False
+        print sys.stderr.write("getPronomSignature(): unexpected return")
+        return False
+    except Exception, e:
+        print sys.stderr.write("getPronomSignature(): unknown error: "+str(e))
+        return False

data/tools/fido/toxml.py

@@ -0,0 +1,52 @@
+#!python
+# -*- coding: utf-8 -*-
+#
+# FIDO csv output to XML
+# Author: Maurice de Rooij <maurice.de.rooij@nationaalarchief.nl>, september 2011
+#
+# Usage in combination with FIDO:
+# Windows: python fido.py [ARGS] | python toxml.py > output.xml
+# Linux: fido.py [ARGS] | toxml.py > output.xml
+#
+# Usage afterwards:
+# Windows: type output.csv | toxml.py > output.xml
+# Linux: cat output.csv | toxml.py > output.xml
+#
+# for difference in usage, see:
+# http://bugs.python.org/issue9390
+# http://support.microsoft.com/default.aspx?kbid=321788
+#
+
+import sys
+import csv
+import string
+
+# define FIDO version
+fidoVersion = '1.0'
+# define PRONOM signature version
+signatureVersion = '56'
+
+sys.stdout.write("""<?xml version="1.0" encoding="utf-8"?>
+<fido_output>
+	<versions>
+		<fido_version>{0}</fido_version>
+		<signature_version>{1}</signature_version>
+	</versions>""".format(fidoVersion,signatureVersion))
+
+reader = csv.reader(sys.stdin)
+
+for row in reader:
+    sys.stdout.write("""
+	<file>
+		<filename>{0}</filename>
+		<status>{1}</status>
+		<matchtype>{2}</matchtype>
+		<time>{3}</time>
+		<puid>{4}</puid>
+		<mimetype>{5}</mimetype>
+		<formatname>{6}</formatname>
+		<signaturename>{7}</signaturename>
+		<filesize>{8}</filesize>
+	</file>""".format(row[6],row[0],row[8],row[1],row[2],row[7],row[3],row[4],row[5]))
+
+sys.stdout.write("\n</fido_output>\n")

data/tools/fido/update_signatures.py

@@ -0,0 +1,171 @@
+#!python
+# -*- coding: utf-8 -*-
+#
+# FIDO SIGNATURE UPDATER  
+#
+# Open Planets Foundation (http://www.openplanetsfoundation.org)
+# See License.txt for license information.
+# Download from: https://github.com/openplanets/fido/releases
+# Author: Maurice de Rooij (NANETH), 2012
+#        
+# FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions .
+# PRONOM is available from http://www.nationalarchives.gov.uk/pronom/
+#
+import sys, os, urllib, time, zipfile, shutil
+
+from xml.etree import ElementTree as CET
+from xml.etree import ElementTree as VET
+from pronomutils import getPronomSignature, checkWellFormedness
+import prepare
+
+defaults = {
+    'version': '1.2.2',
+    'conf_dir': os.path.join(os.path.dirname(__file__), 'conf'),
+    'tmp_dir': 'tmp', 
+    'signatureFileName' : 'DROID_SignatureFile-v{0}.xml',
+    'pronomZipFileName' : 'pronom-xml-v{0}.zip',
+    'fidoSignatureVersion' : 'format_extensions.xml', 
+    'versionsFileName' : 'versions.xml',
+    'http_throttle' : 0.5, # in secs, to prevent DoS of PRONOM server
+    'containerVersion' : 'container-signature-20130501.xml', # container version is frozen and needs human attention before updating
+    'versionXML' : """<?xml version="1.0" encoding="UTF-8"?>\n<versions>\n\t<pronomVersion>{0}</pronomVersion>\n\t<pronomSignature>{1}</pronomSignature>\n\t<pronomContainerSignature>{2}</pronomContainerSignature>\n\t<fidoExtensionSignature>{3}</fidoExtensionSignature>\n\t<updateScript>{4}</updateScript>\n</versions>"""
+    }
+
+def main(defaults):
+    """
+        Updates PRONOM signatures
+        Interactive script, requires keyboard input
+    """
+    try:
+        resume_download = False
+        answers = ['y','yes']
+        versionXML = defaults['versionXML'].format("{0}","{1}",defaults['containerVersion'],defaults['fidoSignatureVersion'],defaults['version'])
+        #print versionXML
+        print "FIDO signature updater v"+defaults['version']
+        print "Contacting PRONOM..."
+        currentVersion = getPronomSignature("version")
+        if currentVersion == False:
+            print "Failed to obtain PRONOM signature file version number, please try again"
+            sys.exit()
+        print "Querying latest signaturefile version..."
+        signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion))
+        if os.path.isfile(signatureFile):
+            print "You already have the latest PRONOM signature file, version "+str(currentVersion)
+            ask = raw_input("Update anyway? (yes/no): ")
+            if ask.lower() not in answers:
+                sys.exit()
+        print "Downloading signature file version "+str(currentVersion)+"..."
+        currentFile = getPronomSignature("file")
+        if currentFile == False:
+            print "Failed to obtain PRONOM signature file, please try again"
+            exit()
+        sigfile = open(signatureFile,'wb')
+        sigfile.write(currentFile)
+        sigfile.close()
+        print "Writing {0}...".format(defaults['signatureFileName'].format(currentVersion))
+        print "Extracting PRONOM PUID's from signature file..."
+        tree = CET.parse(signatureFile)
+        puids = []
+        for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"):
+            puids.append(node.get("PUID"))
+        numberPuids = len(puids)
+        print "Found "+str(numberPuids)+" PRONOM PUID's"
+        print "Downloading signatures can take a while"
+        ask = raw_input("Continue and download signatures? (yes/no): ")
+        if ask.lower() not in answers:
+            print "Aborting update..."
+            sys.exit()
+        tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'])
+        if os.path.isdir(tmpdir):
+            print "Found previously created temporary folder for download:", tmpdir
+            ask = raw_input("Resume download (yes) or start over (no)?: ")
+            if ask.lower() in answers:
+                print "Resuming download..."
+                resume_download = True
+            else:
+                resume_download = False
+        else:
+            print "Creating temporary folder for download:", tmpdir
+            try:
+                os.mkdir(tmpdir)
+            except:
+                pass
+        if not os.path.isdir(tmpdir):
+            tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']))
+            print "Failed to create temporary folder for PUID's, using", tmpdir
+        print "Downloading signatures, one moment please..."
+        one_percent = (float(numberPuids) / 100)
+        numfiles = 0
+        for puid in puids:
+            puidType, puidNum = puid.split("/")
+            puidFileName = "puid."+puidType+"."+puidNum+".xml"
+            filename = os.path.join(tmpdir, puidFileName)
+            if os.path.isfile(filename) and checkWellFormedness(filename) and resume_download is not False:
+                numfiles += 1
+                continue
+            puidUrl = "http://www.nationalarchives.gov.uk/pronom/"+puid+".xml"
+            try:
+                filehandle = urllib.urlopen(puidUrl)
+            except Exception, e:
+                print "Failed to download signaturefile:", puidUrl
+                print "Error:", str(e)
+                print "Please restart and resume download"
+                sys.exit()
+            puidfile = open(filename,'wb')
+            for lines in filehandle.readlines():
+                    puidfile.write(lines)
+            puidfile.close()
+            filehandle.close()
+            if not checkWellFormedness(filename):
+                os.unlink(filename)
+                continue
+            numfiles += 1
+            percent = int(float(numfiles) / one_percent)
+            print "\r",
+            print str(percent)+"%",
+            time.sleep(defaults['http_throttle'])
+        print "100%"
+        try:
+            import zlib
+            compression = zipfile.ZIP_DEFLATED
+        except:
+            compression = zipfile.ZIP_STORED
+        modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'}
+        print "Creating PRONOM zip,",
+        zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFileName'].format(currentVersion)), mode='w')
+        print "adding files with compression mode '"+modes[compression]+"'"
+        for puid in puids:
+            puidType, puidNum = puid.split("/")
+            puidFileName = "puid."+puidType+"."+puidNum+".xml"
+            filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName)
+            if os.path.isfile(filename):
+                zf.write(filename, arcname=puidFileName, compress_type=compression)
+                os.unlink(filename)
+        zf.close()
+        print "Deleting temporary folder and files..."
+        try:
+            for root, dirs, files in os.walk(tmpdir, topdown=False):
+                for name in files:
+                    os.remove(os.path.join(root, name))
+                for name in dirs:
+                    os.rmdir(os.path.join(root, name))
+                os.rmdir(tmpdir)
+        except:
+            pass
+        # update versions.xml
+        versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versionsFileName'])
+        print "Updating {0}...".format(defaults['versionsFileName'])
+        xmlversionsfile = open(versionsFile,'wb')
+        xmlversionsfile.write(versionXML.format(str(currentVersion),"formats-v"+str(currentVersion)+".xml"))
+        xmlversionsfile.close()
+        print "Preparing to convert PRONOM formats to FIDO signatures..."
+        # there should be a check here to handle prepare.main exit() signal (-1/0/1/...)
+        prepare.main()
+        print "FIDO signatures successfully updated"
+        sys.exit()
+    except KeyboardInterrupt:
+        print "\nAborting update"
+        sys.exit()
+
+if __name__ == '__main__':
+    main(defaults)