libis-format 0.9.30 → 0.9.32

data/tools/fido/package.py

@@ -0,0 +1,96 @@
+"""Support for containers."""
+
+import re
+import zipfile
+
+import olefile
+from six import iteritems
+
+
+class Package(object):
+    """Base class for container support."""
+
+    def _process_puid_map(self, data, puid_map):
+        results = []
+        for puid, signatures in iteritems(puid_map):
+            results.extend(self._process_matches(data, puid, signatures))
+
+        return results
+
+    def _process_matches(self, data, puid, signatures):
+        results = []
+        for signature in signatures:
+            if re.search(signature["signature"], data):
+                results.append(puid)
+
+        return results
+
+
+class OlePackage(Package):
+    """OlePackage supports OLE containers."""
+
+    def __init__(self, ole, signatures):
+        """Instantiate OlePackage object given the location of its file and signatures."""
+        self.ole = ole
+        self.signatures = signatures
+
+    def detect_formats(self):
+        """Detect available formats inside the OLE container."""
+        try:
+            ole = olefile.OleFileIO(self.ole)
+        except IOError:
+            return []
+
+        results = []
+        for path, puid_map in iteritems(self.signatures):
+            # Each OLE container signature lists the path of the file inside the OLE
+            # on which it operates; if the file is missing, there can be no match.
+            # This is not a precise match because the name of the stream may slightly
+            # differ; for example, \x01CompObj instead of CompObj
+            filepath = None
+            for paths in ole.listdir():
+                p = '/'.join(paths)
+                if p == path or p[1:] == path:
+                    filepath = p
+                    break
+
+            # Path to match isn't in the container at all
+            if filepath is None:
+                continue
+
+            with ole.openstream(filepath) as stream:
+                contents = stream.read()
+                results.extend(self._process_puid_map(contents, puid_map))
+
+        return results
+
+
+class ZipPackage(Package):
+    """ZipPackage supports Zip containers."""
+
+    def __init__(self, zip_, signatures):
+        """Instantiate ZipPackage object given the location of its file and signatures."""
+        self.zip = zip_
+        self.signatures = signatures
+
+    def detect_formats(self):
+        """Detect available formats inside the ZIP container."""
+        try:
+            zip_ = zipfile.ZipFile(self.zip)
+        except (zipfile.BadZipfile, RuntimeError, UnicodeDecodeError):
+            return []
+
+        results = []
+        for path, puid_map in iteritems(self.signatures):
+            # Each ZIP container signature lists the path of the file inside the ZIP
+            # on which it operates; if the file is missing, there can be no match.
+            if path not in zip_.namelist():
+                continue
+
+            # Extract the requested file from the ZIP only once, and pass the same
+            # data to each signature that requires it.
+            with zip_.open(path) as id_file:
+                contents = id_file.read()
+                results.extend(self._process_puid_map(contents, puid_map))
+
+        return results

data/tools/fido/prepare.py

@@ -1,88 +1,112 @@
-#!python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Format Identification for Digital Objects
 
-# MdR: 'reload(sys)' and 'setdefaultencoding("utf-8")' needed to fix utf-8 encoding errors
-# when converting from PRONOM to FIDO format
-import sys
-reload(sys)
-sys.setdefaultencoding("utf-8")
-import cStringIO, zipfile, os
+"""Format Identification for Digital Objects."""
+
+from __future__ import print_function
+
+from argparse import ArgumentParser
 import hashlib
-import urllib
+import sys
+from xml.dom import minidom
 from xml.etree import ElementTree as ET
-from xml.etree import ElementTree as VET # versions.xml
-# needed for debug
-# print_r: https://github.com/marcbelmont/python-print_r
-# from print_r import print_r
+import zipfile
+
+from six.moves import cStringIO
+from six.moves.urllib.request import urlopen
+from six.moves.urllib.parse import urlparse
+
+from .pronomutils import get_local_pronom_versions
+
+
+# \a\b\n\r\t\v
+# MdR: took out '<' and '>' out of _ordinary because they were converted to entities &lt;&gt;
+# MdR: moved '!' from _ordinary to _special because it means "NOT" in the regex world. At this time no regex in any sig has a negate set, did this to be on the safe side
+_ordinary = frozenset(' "#%&\',-/0123456789:;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~')
+_special = '$()*+.?![]^\\{|}'
+_hex = '0123456789abcdef'
+
 
 class NS:
-    """Helper class for XML name spaces in ElementTree.
-       Use like MYNS=NS("{http://some/uri}") and then
-       MYNS(tag1/tag2).
     """
+    Helper class for XML name spaces in ElementTree.
+
+    Use like MYNS=NS("{http://some/uri}") and then MYNS(tag1/tag2).
+    """
+
     def __init__(self, uri):
+        """Instantiate class with `uri` argument."""
         self.uri = uri
+
     def __getattr__(self, tag):
+        """Append URI to the class attributes."""
         return self.uri + tag
+
     def __call__(self, path):
+        """Define behavior when the instant is used as a function."""
         return "/".join(getattr(self, tag) for tag in path.split("/"))
 
-# XHTML namespace
-XHTML = NS("{http://www.w3.org/1999/xhtml}")
-# TNA namespace
-TNA = NS("{http://pronom.nationalarchives.gov.uk}")
+
+XHTML = NS("{http://www.w3.org/1999/xhtml}")  # XHTML namespace
+TNA = NS("{http://pronom.nationalarchives.gov.uk}")  # TNA namespace
+
 
 def get_text_tna(element, tag, default=''):
-    """Helper function to return the text for a tag or path using the TNA namespace.
-    """
+    """Helper function to return the text for a tag or path using the TNA namespace."""
     part = element.find(TNA(tag))
-    return part.text.strip() if part != None and part.text != None else default
+    if part is None or part.text is None:
+        return default
+    return part.text.strip()
+
 
 def prettify(elem):
-    """Return a pretty-printed XML string for the Element.
-    """
-    from xml.dom import minidom
+    """Return a pretty-printed XML string for the Element."""
     rough_string = ET.tostring(elem, 'UTF-8')
     reparsed = minidom.parseString(rough_string)
     return reparsed.toprettyxml(indent="  ")
 
+
 class FormatInfo:
+    """Convert PRONOM formats into FIDO signatures."""
+
     def __init__(self, pronom_files, format_list=[]):
+        """Instantiate class, take a list of PRONOM files and an optional list of formats."""
         self.info = {}
         self.formats = []
         self.pronom_files = pronom_files
         for f in format_list:
-            self.add_format(f)
-                             
-    def save(self, dst):
-        """Write the fido XML format definitions to @param dst
-        """
-        tree = ET.ElementTree(ET.Element('formats', {'version':'0.3',
-                                                     'xmlns:xsi' : "http://www.w3.org/2001/XMLSchema-instance",
-                                                     'xsi:noNamespaceSchemaLocation': "fido-formats.xsd",
-                                                     'xmlns:dc': "http://purl.org/dc/elements/1.1/",
-                                                     'xmlns:dcterms': "http://purl.org/dc/terms/"}))
+            self.add_format(f)  # FIXME: add_format is undefined!
+
+    def save(self, dst=sys.stdout):
+        """Write the fido XML format definitions to @param dst."""
+        tree = ET.ElementTree(ET.Element('formats', {
+            'version': '0.3',
+            'xmlns:xsi': "http://www.w3.org/2001/XMLSchema-instance",
+            'xsi:noNamespaceSchemaLocation': "fido-formats.xsd",
+            'xmlns:dc': "http://purl.org/dc/elements/1.1/",
+            'xmlns:dcterms': "http://purl.org/dc/terms/"
+        }))
         root = tree.getroot()
         for f in self.formats:
             # MdR: this skipped puids without sig, but we want them ALL
             # because puid might be matched on extension
-            #if f.find('signature'):
+            # if f.find('signature'):
             root.append(f)
         self.indent(root)
-        with open(dst, 'wb') as out:
-                #print >>out, ET.tostring(root,encoding='utf-8')     
-                print >>out, ET.tostring(root)     
+        with open(dst, 'wb') as file_:
+            # print >>out, ET.tostring(root,encoding='utf-8')
+            print(ET.tostring(root), file=file_)
 
     def indent(self, elem, level=0):
-        i = "\n" + level*"  "
+        """Indent output."""
+        i = "\n" + level * "  "
         if len(elem):
             if not elem.text or not elem.text.strip():
                 elem.text = i + "  "
             if not elem.tail or not elem.tail.strip():
                 elem.tail = i
             for elem in elem:
-                self.indent(elem, level+1)
+                self.indent(elem, level + 1)
             if not elem.tail or not elem.tail.strip():
                 elem.tail = i
         else:
@@ -90,52 +114,57 @@ class FormatInfo:
                 elem.tail = i
 
     def load_pronom_xml(self, puid_filter=None):
-        """Load the pronom XML from self.pronom_files and convert it to fido XML.
-           As a side-effect, set self.formats to a list of ElementTree.Element
-           If a @param puid is specified, only that one will be loaded.
+        """
+        Load the pronom XML from self.pronom_files and convert it to fido XML.
+
+        As a side-effect, set self.formats to a list of ElementTree.Element.
+        If a @param puid is specified, only that one will be loaded.
         """
         formats = []
-        #for p in self.pronom_files:
+        # for p in self.pronom_files:
         #    print p
-        #print self.pronom_files
-        #exit()
+        # print self.pronom_files
+        # exit()
         try:
             zip = zipfile.ZipFile(self.pronom_files, 'r')
             for item in zip.infolist():
-                #print item.filename
+                # print item.filename
                 try:
                     stream = zip.open(item)
                     # Work is done here!
-                    #if item.filename != 'github/fido/fido/conf/pronom-xml/puid.fmt.11.xml':
-                    format = self.parse_pronom_xml(stream, puid_filter)
-                    if format != None:
-                        formats.append(format) 
+                    # if item.filename != 'github/fido/fido/conf/pronom-xml/puid.fmt.11.xml':
+                    format_ = self.parse_pronom_xml(stream, puid_filter)
+                    if format_ is not None:
+                        formats.append(format_)
                 finally:
                     stream.close()
         finally:
             try:
                 zip.close()
-            except Exception, e:
-                sys.stderr.write("An error occured loading '{0}' (exception: {1})".format(self.pronom_files, e))
+            except Exception as e:
+                print("An error occured loading '{0}' (exception: {1})".format(self.pronom_files, e), file=sys.stderr)
                 sys.exit()
         # Replace the formatID with puids in has_priority_over
-        id_map = {}
-        for element in formats:
-            puid = element.find('puid').text
-            #print "working on puid:",puid
-            pronom_id = element.find('pronom_id').text
-            id_map[pronom_id] = puid
-        for element in formats:
-            for rel in element.findall('has_priority_over'):
-                rel.text = id_map[rel.text]
+        if puid_filter is None:
+            id_map = {}
+            for element in formats:
+                puid = element.find('puid').text
+                # print "working on puid:",puid
+                pronom_id = element.find('pronom_id').text
+                id_map[pronom_id] = puid
+            for element in formats:
+                for rel in element.findall('has_priority_over'):
+                    rel.text = id_map[rel.text]
 
         self._sort_formats(formats)
         self.formats = formats
-                    
+
     def parse_pronom_xml(self, source, puid_filter=None):
-        """Read a pronom XML from @param source, convert it to fido XML and
-           @return ET.ElementTree Element representing it.
-           If a @param puid is specified, only that one will be loaded.
+        """
+        Parse PRONOM XML and convert into FIDO XML.
+
+        If a @param puid is specified, only that one will be loaded.
+        @return ET.ElementTree Element representing it.
         """
         pronom_xml = ET.parse(source)
         pronom_root = pronom_xml.getroot()
@@ -147,13 +176,13 @@ class FormatInfo:
             if type == 'PUID':
                 puid = get_text_tna(id, 'Identifier')
                 ET.SubElement(fido_format, 'puid').text = puid
-                if puid_filter != None and puid != puid_filter:
+                if puid_filter and puid != puid_filter:
                     return None
         # A bit clumsy.  I want to have puid first, then mime, then container.
         for id in pronom_format.findall(TNA('FileFormatIdentifier')):
             type = get_text_tna(id, 'IdentifierType')
             if type == 'MIME':
-                ET.SubElement(fido_format, 'mime').text = get_text_tna(id, 'Identifier')  
+                ET.SubElement(fido_format, 'mime').text = get_text_tna(id, 'Identifier')
             elif type == 'PUID':
                 puid = get_text_tna(id, 'Identifier')
                 if puid == 'x-fmt/263':
@@ -170,7 +199,7 @@ class FormatInfo:
         for id in pronom_format.findall(TNA('FileFormatIdentifier')):
             type = get_text_tna(id, 'IdentifierType')
             if type == 'Apple Uniform Type Identifier':
-                ET.SubElement(fido_format, 'apple_uid').text = get_text_tna(id, 'Identifier')  
+                ET.SubElement(fido_format, 'apple_uid').text = get_text_tna(id, 'Identifier')
         # Handle the relationships
         for x in pronom_format.findall(TNA('RelatedFormat')):
             rel = get_text_tna(x, 'RelationshipType')
@@ -181,20 +210,20 @@ class FormatInfo:
             fido_sig = ET.SubElement(fido_format, 'signature')
             ET.SubElement(fido_sig, 'name').text = get_text_tna(pronom_sig, 'SignatureName')
             # There are some funny chars in the notes, which caused me trouble and it is a unicode string,
-            ET.SubElement(fido_sig, 'note').text = get_text_tna(pronom_sig, 'SignatureNote').encode('UTF-8')
+            ET.SubElement(fido_sig, 'note').text = get_text_tna(pronom_sig, 'SignatureNote')
             for pronom_pat in pronom_sig.findall(TNA('ByteSequence')):
                 fido_pat = ET.SubElement(fido_sig, 'pattern')
                 pos = fido_position(get_text_tna(pronom_pat, 'PositionType'))
                 bytes = get_text_tna(pronom_pat, 'ByteSequenceValue')
                 offset = get_text_tna(pronom_pat, 'Offset')
                 max_offset = get_text_tna(pronom_pat, 'MaxOffset')
-                if max_offset == None:
+                if not max_offset:
                     pass
-                #print "working on puid:", puid, ", position: ", pos, "with offset, maxoffset: ", offset, ",", max_offset
+                # print "working on puid:", puid, ", position: ", pos, "with offset, maxoffset: ", offset, ",", max_offset
                 regex = convert_to_regex(bytes, 'Little', pos, offset, max_offset)
-                #print "done puid", puid
+                # print "done puid", puid
                 if regex == "__INCOMPATIBLE_SIG__":
-                    print >> sys.stderr, "Error: incompatible PRONOM signature found for puid", puid, ", skipping..."
+                    print("Error: incompatible PRONOM signature found for puid {} skipping...".format(puid), file=sys.stderr)
                     # remove the empty 'signature' nodes
                     # now that the signature is not compatible and thus "regex" is empty
                     remove = fido_format.findall('signature')
@@ -205,8 +234,8 @@ class FormatInfo:
                 ET.SubElement(fido_pat, 'pronom_pattern').text = bytes
                 ET.SubElement(fido_pat, 'regex').text = regex
         # Get the format details
-        fido_details = ET.SubElement(fido_format,'details')
-        ET.SubElement(fido_details, 'dc:description').text = get_text_tna(pronom_format, 'FormatDescription').encode('utf8')
+        fido_details = ET.SubElement(fido_format, 'details')
+        ET.SubElement(fido_details, 'dc:description').text = get_text_tna(pronom_format, 'FormatDescription')
         ET.SubElement(fido_details, 'dcterms:available').text = get_text_tna(pronom_format, 'ReleaseDate')
         ET.SubElement(fido_details, 'dc:creator').text = get_text_tna(pronom_format, 'Developers/DeveloperCompoundName')
         ET.SubElement(fido_details, 'dcterms:publisher').text = get_text_tna(pronom_format, 'Developers/OrganisationName')
@@ -221,7 +250,7 @@ class FormatInfo:
         ET.SubElement(fido_details, 'content_type').text = get_text_tna(pronom_format, 'FormatTypes')
         # References
         for x in pronom_format.findall(TNA("Document")):
-            r = ET.SubElement(fido_details,'reference')
+            r = ET.SubElement(fido_details, 'reference')
             ET.SubElement(r, 'dc:title').text = get_text_tna(x, 'TitleText')
             ET.SubElement(r, 'dc:creator').text = get_text_tna(x, 'Author/AuthorCompoundName')
             ET.SubElement(r, 'dc:publisher').text = get_text_tna(x, 'Publisher/PublisherCompoundName')
@@ -229,51 +258,53 @@ class FormatInfo:
             for id in x.findall(TNA('DocumentIdentifier')):
                 type = get_text_tna(id, 'IdentifierType')
                 if type == 'URL':
-                    ET.SubElement(r, 'dc:identifier').text = "http://"+get_text_tna(id, 'Identifier')  
+                    ET.SubElement(r, 'dc:identifier').text = "http://" + get_text_tna(id, 'Identifier')
                 else:
-                    ET.SubElement(r, 'dc:identifier').text = get_text_tna(id, 'IdentifierType')+":"+get_text_tna(id, 'Identifier')  
+                    ET.SubElement(r, 'dc:identifier').text = get_text_tna(id, 'IdentifierType') + ":" + get_text_tna(id, 'Identifier')
             ET.SubElement(r, 'dc:description').text = get_text_tna(x, 'DocumentNote')
             ET.SubElement(r, 'dc:type').text = get_text_tna(x, 'DocumentType')
-            ET.SubElement(r, 'dcterms:license').text = get_text_tna(x, 'AvailabilityDescription')+" "+get_text_tna(x, 'AvailabilityNote')
+            ET.SubElement(r, 'dcterms:license').text = get_text_tna(x, 'AvailabilityDescription') + " " + get_text_tna(x, 'AvailabilityNote')
             ET.SubElement(r, 'dc:rights').text = get_text_tna(x, 'DocumentIPR')
-#         Examples
+        # Examples
         for x in pronom_format.findall(TNA("ReferenceFile")):
-            rf = ET.SubElement(fido_details,'example_file')
+            rf = ET.SubElement(fido_details, 'example_file')
             ET.SubElement(rf, 'dc:title').text = get_text_tna(x, 'ReferenceFileName')
             ET.SubElement(rf, 'dc:description').text = get_text_tna(x, 'ReferenceFileDescription')
             checksum = ""
             for id in x.findall(TNA('ReferenceFileIdentifier')):
                 type = get_text_tna(id, 'IdentifierType')
                 if type == 'URL':
-                    url = "http://"+get_text_tna(id, 'Identifier')
-                    ET.SubElement(rf, 'dc:identifier').text = url  
+                    # Starting with PRONOM 89, some URLs contain http://
+                    # and others do not.
+                    url = get_text_tna(id, 'Identifier')
+                    if not urlparse(url).scheme:
+                        url = "http://" + url
+                    ET.SubElement(rf, 'dc:identifier').text = url
                     # And calculate the checksum of this resource:
                     m = hashlib.md5()
-                    sock = urllib.urlopen(url)
+                    sock = urlopen(url)
                     m.update(sock.read())
                     sock.close()
-                    checksum=m.hexdigest()
+                    checksum = m.hexdigest()
                 else:
-                    ET.SubElement(rf, 'dc:identifier').text = get_text_tna(id, 'IdentifierType')+":"+get_text_tna(id, 'Identifier')  
+                    ET.SubElement(rf, 'dc:identifier').text = get_text_tna(id, 'IdentifierType') + ":" + get_text_tna(id, 'Identifier')
             ET.SubElement(rf, 'dcterms:license').text = ""
             ET.SubElement(rf, 'dc:rights').text = get_text_tna(x, 'ReferenceFileIPR')
             checksumElement = ET.SubElement(rf, 'checksum')
             checksumElement.text = checksum
             checksumElement.attrib['type'] = "md5"
         # Record Metadata
-        md = ET.SubElement(fido_details,'record_metadata')
-        ET.SubElement(md, 'status').text ='unknown'
+        md = ET.SubElement(fido_details, 'record_metadata')
+        ET.SubElement(md, 'status').text = 'unknown'
         ET.SubElement(md, 'dc:creator').text = get_text_tna(pronom_format, 'ProvenanceName')
         ET.SubElement(md, 'dcterms:created').text = get_text_tna(pronom_format, 'ProvenanceSourceDate')
         ET.SubElement(md, 'dcterms:modified').text = get_text_tna(pronom_format, 'LastUpdatedDate')
-        ET.SubElement(md, 'dc:description').text = get_text_tna(pronom_format, 'ProvenanceDescription').encode('utf8')
-        return fido_format  
-        
-    #FIXME: I don't think that this quite works yet!
+        ET.SubElement(md, 'dc:description').text = get_text_tna(pronom_format, 'ProvenanceDescription')
+        return fido_format
+
+    # FIXME: I don't think that this quite works yet!
     def _sort_formats(self, formatlist):
-        """Sort the format list based on their priority relationships so higher priority
-           formats appear earlier in the list.
-        """
+        """Sort the format list based on their priority relationships so higher priority formats appear earlier in the list."""
         def compare_formats(f1, f2):
             f1ID = f1.find('puid').text
             f2ID = f2.find('puid').text
@@ -291,9 +322,9 @@ class FormatInfo:
                 return 1
         return sorted(formatlist, cmp=compare_formats)
 
+
 def fido_position(pronom_position):
-    """@return BOF/EOF/VAR instead of the more verbose pronom position names.
-    """
+    """Return BOF/EOF/VAR instead of the more verbose pronom position names."""
     if pronom_position == 'Absolute from BOF':
         return 'BOF'
     elif pronom_position == 'Absolute from EOF':
@@ -302,16 +333,20 @@ def fido_position(pronom_position):
         return 'VAR'
     elif pronom_position == 'Indirect From BOF':
         return 'IFB'
-    else: # to make sure FIDO does not crash (IFB aftermath)
-        sys.stderr.write("Unknown pronom PositionType:" + pronom_position)    
+    else:  # to make sure FIDO does not crash (IFB aftermath)
+        sys.stderr.write("Unknown pronom PositionType:" + pronom_position)
         return 'VAR'
 
+
 def _convert_err_msg(msg, c, i, chars):
     return "Conversion: {0}: char='{1}', at pos {2} in \n  {3}\n  {4}^\nBuffer = {5}".format(msg, c, i, chars, i * ' ', buf.getvalue())
 
+
 def doByte(chars, i, littleendian):
-    """Convert two chars[i] and chars[i+1] into a byte.  
-       @return a tuple (byte, 2) 
+    """
+    Convert two chars[i] and chars[i+1] into a byte.
+
+    @return a tuple (byte, 2)
     """
     c1 = '0123456789ABCDEF'.find(chars[i].upper())
     c2 = '0123456789ABCDEF'.find(chars[i + 1].upper())
@@ -323,12 +358,7 @@ def doByte(chars, i, littleendian):
         val = chr(c1 + 16 * c2)
     return (escape(val), 2)
 
-# \a\b\n\r\t\v
-# MdR: took out '<' and '>' out of _ordinary because they were converted to entities &lt;&gt;
-# MdR: moved '!' from _ordinary to _special because it means "NOT" in the regex world. At this time no regex in any sig has a negate set, did this to be on the safe side
-_ordinary = frozenset(' "#%&\',-/0123456789:;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~')
-_special = '$()*+.?![]^\\{|}'
-_hex = '0123456789abcdef'
+
 def _escape_char(c):
     if c in '\n':
         return '\\n'
@@ -340,68 +370,66 @@ def _escape_char(c):
         (high, low) = divmod(ord(c), 16)
         return '\\x' + _hex[high] + _hex[low]
 
+
 def escape(string):
-    "Escape characters in pattern that are non-printable, non-ascii, or special for regexes."
+    """Escape characters in pattern that are non-printable, non-ascii, or special for regexes."""
     return ''.join(c if c in _ordinary else _escape_char(c) for c in string)
 
+
 def calculate_repetition(char, pos, offset, maxoffset):
-    """
-    Recursively calculates offset/maxoffset repetition,
-    when one or both offsets is greater than 65535 bytes (64KB)
-    see: bugs.python.org/issue13169
-    Otherwise it returns the {offset,maxoffset}
-    """
-    calcbuf = cStringIO.StringIO()
-    
+    """Recursively calculates offset/maxoffset repetition, when one or both offsets is greater than 65535 bytes (64KB). See: https://bugs.python.org/issue13169."""
+    calcbuf = cStringIO()
+
     calcremain = False
     offsetremain = 0
     maxoffsetremain = 0
-    
-    if offset != None and offset != '':
-        if int(offset) > 65535:
-            offsetremain = str(int(offset) - 65535)
-            offset = '65535'
-            calcremain = True
-    if maxoffset != None and maxoffset != '':
-        if int(maxoffset) > 65535:
-            maxoffsetremain = str(int(maxoffset) - 65535)
-            maxoffset = '65535'
-            calcremain = True
-    
+
+    if offset is not None and int(offset) > 65535:
+        offsetremain = str(int(offset) - 65535)
+        offset = '65535'
+        calcremain = True
+    if maxoffset is not None and int(maxoffset) > 65535:
+        maxoffsetremain = str(int(maxoffset) - 65535)
+        maxoffset = '65535'
+        calcremain = True
+
     if pos == "BOF" or pos == "EOF":
         if offset != '0':
             calcbuf.write(char + '{' + str(offset))
-            if maxoffset != None:
+            if maxoffset is not None:
                 calcbuf.write(',' + maxoffset)
             calcbuf.write('}')
-        elif maxoffset != None:
+        elif maxoffset is not None:
             calcbuf.write(char + '{0,' + maxoffset + '}')
 
     if pos == "IFB":
         if offset != '0':
             calcbuf.write(char + '{' + str(offset))
-            if maxoffset != None:
+            if maxoffset is not None:
                 calcbuf.write(',' + maxoffset)
             calcbuf.write('}')
-            if maxoffset == None:
+            if maxoffset is not None:
                 calcbuf.write(',}')
-        elif maxoffset != None:
+        elif maxoffset is not None:
             calcbuf.write(char + '{0,' + maxoffset + '}')
 
-    if calcremain: # recursion happens here
+    if calcremain:  # recursion happens here
         calcbuf.write(calculate_repetition(char, pos, offsetremain, maxoffsetremain))
-    
+
     val = calcbuf.getvalue()
     calcbuf.close()
     return val
 
+
 def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
-    """Convert 
-       @param chars, a pronom bytesequence, into a 
-       @return regular expression.
-       Endianness is not used.
     """
+    Convert to regular expression.
+
+    Endianness is not used.
 
+    @param chars, a pronom bytesequence, into a
+    @return regular expression.
+    """
     if 'Big' in endianness:
         littleendian = False
     else:
@@ -410,24 +438,26 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
         offset = '0'
     if len(maxoffset) == 0:
         maxoffset = None
+    if maxoffset == '0':
+        maxoffset = None
     # make buf global so we can print it @'_convert_err_msg' while debugging (MdR)
     global buf
-    buf = cStringIO.StringIO()
-    buf.write("(?s)")   #If a regex starts with (?s), it is equivalent to DOTALL.   
+    buf = cStringIO()
+    buf.write("(?s)")  # If a regex starts with (?s), it is equivalent to DOTALL.
     i = 0
     state = 'start'
     if 'BOF' in pos:
-        buf.write('\\A') # start of regex
+        buf.write('\\A')  # start of regex
         buf.write(calculate_repetition('.', pos, offset, maxoffset))
-            
+
     if 'IFB' in pos:
         buf.write('\\A')
         buf.write(calculate_repetition('.', pos, offset, maxoffset))
-            
+
     while True:
         if i == len(chars):
             break
-        #print _convert_err_msg(state,chars[i],i,chars)
+        # print _convert_err_msg(state,chars[i],i,chars)
         if state == 'start':
             if chars[i].isalnum():
                 state = 'bytes'
@@ -471,7 +501,7 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
                 (byt, inc) = doByte(chars, i, littleendian)
                 buf.write(byt)
                 i += inc
-                #assert(chars[i] == ':')
+                # assert(chars[i] == ':')
                 if chars[i] != ':':
                     return "__INCOMPATIBLE_SIG__"
                 buf.write('-')
@@ -479,13 +509,13 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
                 (byt, inc) = doByte(chars, i, littleendian)
                 buf.write(byt)
                 i += inc
-                #assert(chars[i] == ']')
+                # assert(chars[i] == ']')
                 if chars[i] != ']':
                     return "__INCOMPATIBLE_SIG__"
                 buf.write(']')
                 i += 1
             except Exception:
-                print _convert_err_msg('Illegal character in bracket', chars[i], i, chars)
+                print(_convert_err_msg('Illegal character in bracket', chars[i], i, chars))
                 raise
             if i < len(chars) and chars[i] == '{':
                 state = 'curly-after-bracket'
@@ -511,7 +541,7 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
                     (byt, inc) = doByte(chars, i, littleendian)
                     buf.write(byt)
                     i += inc
-                    #assert(chars[i] == ':')
+                    # assert(chars[i] == ':')
                     if chars[i] != ':':
                         return "__INCOMPATIBLE_SIG__"
                     buf.write('-')
@@ -519,8 +549,8 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
                     (byt, inc) = doByte(chars, i, littleendian)
                     buf.write(byt)
                     i += inc
-    
-                    #assert(chars[i] == ']')
+
+                    # assert(chars[i] == ']')
                     if chars[i] != ']':
                         return "__INCOMPATIBLE_SIG__"
                     buf.write(']')
@@ -537,7 +567,7 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
             # when there is a curly-after-bracket, then the {m,n} applies to the bracketed item
             # The above, while sensible, appears to be incorrect.  A '.' is always needed.
             # for droid equiv behavior
-            #if state == 'curly':
+            # if state == 'curly':
             buf.write('.')
             buf.write('{')
             i += 1                # skip the (
@@ -548,7 +578,7 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
                 elif chars[i] == '-':
                     buf.write(',')
                     i += 1
-                elif chars[i] == '*': # skip the *
+                elif chars[i] == '*':  # skip the *
                     i += 1
                 elif chars[i] == '}':
                     break
@@ -581,36 +611,35 @@ def convert_to_regex(chars, endianness='', pos='BOF', offset='0', maxoffset=''):
     buf.close()
     return val
 
-def main(arg=None):
-    import sys
-    from argparselocal import ArgumentParser
-    if arg != None:
-        arglist = arg
-    else:
-        arglist = sys.argv[1:]
-#    print arglist
-#    exit()
-    mydir = os.path.abspath(os.path.dirname(__file__))
-    # parse version file to fetch versions
-    versionsFile = os.path.join(mydir, 'conf', 'versions.xml')
-    try:
-        versions = VET.parse(versionsFile)
-    except Exception, e:
-        sys.stderr.write("An error occured loading versions.xml:\n{0}".format(e))
-        sys.exit()
-    xml_pronomSignature = os.path.join(mydir, 'conf', versions.find('pronomSignature').text)
-    xml_pronomZipFile = os.path.join(mydir, 'conf', "pronom-xml-v{0}.zip".format(versions.find('pronomVersion').text))
-    parser = ArgumentParser(description='Produce the fido format xml that is loaded at run-time')
-    parser.add_argument('-input', default=xml_pronomZipFile, help='input file, a zip containing Pronom xml files')
-    parser.add_argument('-output', default=xml_pronomSignature, help='output file')
-    parser.add_argument('-puid', default=None, help='a particular PUID record to extract')
-    # PROCESS ARGUMENTS
-    args = parser.parse_args(arglist)
-    # print os.path.abspath(args.input), os.path.abspath(args.output)
-    info = FormatInfo(args.input)
-    info.load_pronom_xml(args.puid)
-    info.save(args.output)
-    print >> sys.stderr, 'Converted {0} PRONOM formats to FIDO signatures'.format(len(info.formats))
-    
+
+def run(input=None, output=None, puid=None):
+    """Convert PRONOM formats into FIDO signatures."""
+    versions = get_local_pronom_versions()
+
+    if input is None:
+        input = versions.get_zip_file()
+    if output is None:
+        output = versions.get_signature_file()
+
+    info = FormatInfo(input)
+    info.load_pronom_xml(puid)
+    info.save(output)
+    print('Converted {0} PRONOM formats to FIDO signatures'.format(len(info.formats)), file=sys.stderr)
+
+
+def main(args=None):
+    """Main CLI entrypoint."""
+    if args is None:
+        args = sys.argv[1:]
+
+    parser = ArgumentParser(description='Produce the FIDO format XML that is loaded at run-time')
+    parser.add_argument('-input', default=None, help='Input file, a Zip containing PRONOM XML files')
+    parser.add_argument('-output', default=None, help='Ouptut file')
+    parser.add_argument('-puid', default=None, help='A particular PUID record to extract')
+    args = parser.parse_args(args)
+
+    run(input=args.input, output=args.output, puid=args.puid)
+
+
 if __name__ == '__main__':
-    main()    
+    main()