libis-format 0.9.5-java

data/tools/fido/fido.py

@@ -0,0 +1,854 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import sys, re, os, time, math
+import hashlib, urllib, urlparse, csv, getopt
+from xml.etree import cElementTree as ET
+from xml.etree import ElementTree as CET
+from xml.etree import ElementTree as VET # versions.xml
+
+version = '1.3.1'
+defaults = {'bufsize': 128 * 1024, # (bytes)
+            'regexcachesize' :2084, # (bytes)
+            'conf_dir' : os.path.join(os.path.dirname(__file__), 'conf'),
+            'printmatch': "OK,%(info.time)s,%(info.puid)s,\"%(info.formatname)s\",\"%(info.signaturename)s\",%(info.filesize)s,\"%(info.filename)s\",\"%(info.mimetype)s\",\"%(info.matchtype)s\"\n",
+            'printnomatch' : "KO,%(info.time)s,,,,%(info.filesize)s,\"%(info.filename)s\",,\"%(info.matchtype)s\"\n",
+            'format_files': ['formats-v75.xml', 'format_extensions.xml'],
+            'containersignature_file' : 'container-signature-20150307.xml',
+            # versions.xml is where fido.py reads version information
+            # about which xml to load
+            'versions_file' : 'versions.xml',
+            'container_bufsize' : 512 * 1024, # (bytes)
+            'description' : """
+    Format Identification for Digital Objects (fido).
+    FIDO is a command-line tool to identify the file formats of digital objects.
+    It is designed for simple integration into automated work-flows.
+    """,
+    'epilog' : """
+    Open Planets Foundation (http://www.openplanetsfoundation.org)
+    See License.txt for license information.
+    Download from: https://github.com/openplanets/fido/releases
+    Usage guide: http://wiki.opf-labs.org/display/KB/FIDO+usage+guide 
+    Author: Adam Farquhar (BL), 2010
+    Maintainer: Maurice de Rooij (OPF/NANETH), 2011, 2012, 2013
+    FIDO uses the UK National Archives (TNA) PRONOM File Format
+    and Container descriptions.
+    PRONOM is available from http://www.nationalarchives.gov.uk/pronom/"""
+}
+
+class Fido:
+    def __init__(self, quiet=False, bufsize=None, container_bufsize = None, printnomatch=None, printmatch=None, zip=False, nocontainer=False, handle_matches=None, conf_dir=None, format_files=None, containersignature_file=None):
+        global defaults
+        self.quiet = quiet
+        self.bufsize = (defaults['bufsize'] if bufsize == None else bufsize)
+        self.container_bufsize = (defaults['container_bufsize'] if container_bufsize == None else container_bufsize)
+        self.printmatch = (defaults['printmatch'] if printmatch == None else printmatch)
+        self.printnomatch = (defaults['printnomatch'] if printnomatch == None else printnomatch)
+        self.handle_matches = (self.print_matches if handle_matches == None else handle_matches)
+        self.zip = zip
+        self.nocontainer = (defaults['nocontainer'] if nocontainer == None else nocontainer)
+        self.conf_dir = defaults['conf_dir'] if conf_dir == None else conf_dir
+#        print defaults
+#        sys.exit()
+        self.format_files = defaults['format_files'] if format_files == None else format_files
+        #self.containersignature_file = defaults['containersignature_file'] if containersignature_file == None else containersignature_file
+        self.containersignature_file = defaults['containersignature_file'] #if containersignature_file == None else containersignature_file
+        self.formats = []
+        self.puid_format_map = {}
+        self.puid_has_priority_over_map = {}
+        # load signatures
+        for xml_file in self.format_files:
+            self.load_fido_xml(os.path.join(os.path.abspath(self.conf_dir), xml_file))
+        self.load_container_signature(os.path.join(os.path.abspath(self.conf_dir), self.containersignature_file))
+        self.current_file = ''
+        self.current_filesize = 0
+        self.current_format = None
+        self.current_sig = None
+        self.current_pat = None
+        self.current_count = 0  # Count of calls to match_formats
+        re._MAXCACHE = defaults['regexcachesize']
+        self.externalsig = ET.XML('<signature><name>External</name></signature>')
+
+    _ordinary = frozenset(' "#%&\',-/0123456789:;=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~')
+    #_special = '$*+.?![]^\\{|}'
+    _special = '$()*+.?![]^\\{|}'
+    _hex = '0123456789abcdef'
+    def _escape_char(self,c):
+        if c in '\n':
+            return '\\n'
+        elif c == '\r':
+            return '\\r'
+        elif c in self._special:
+            return '\\' + c
+        else:
+            (high, low) = divmod(ord(c), 16)
+            return '\\x' + self._hex[high] + self._hex[low]
+    
+    def escape(self,string):
+        "Escape characters in pattern that are non-printable, non-ascii, or special for regexes."
+        escaped = ''.join(c if c in self._ordinary else self._escape_char(c) for c in string)
+        return escaped
+
+    def convert_container_sequence(self,sig):
+        """Parse the PRONOM container sequences
+        and convert to regular expressions
+        """
+        seq = '(?s)'
+        inq = False
+        byt = False
+        rng = False
+        ror = False
+        for i in range(len(sig)):
+            if not inq and not rng:
+                if sig[i] == "'":
+                    inq = True
+                    continue
+                if sig[i] == " ":
+                    continue
+                if sig[i] == "[":
+                    seq += "("
+                    rng = True
+                    continue
+                if not byt:
+                    seq += "\\x" + sig[i].lower()
+                    byt = True
+                    continue
+                if byt:
+                    seq += sig[i].lower()
+                    byt = False
+                    continue
+            if inq:
+                if sig[i] == "'" and not rng:
+                    inq = False
+                    continue
+                seq += self.escape(sig[i])
+                continue
+            if rng:
+                if sig[i] == "]":
+                    seq += ")"
+                    rng = False
+                    continue
+                if sig[i] != "-" and sig[i] != "'" and ror:
+                    seq += self.escape(sig[i])
+                    continue
+                if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and not ror and not byt:
+                    seq += "\\x" + sig[i].lower()
+                    byt = True
+                    continue
+                if sig[i] != "-" and sig[i] != "'" and sig[i] != " " and not ror and byt:
+                    seq += sig[i].lower()
+                    byt = False
+                    continue
+                if sig[i] == "-" or sig[i] == " ":
+                    seq += "|"
+                    continue
+                if sig[i] == "'" and not ror:
+                    ror = True
+                    continue
+                if sig[i] == "'" and ror:
+                    ror = False
+                    continue
+        #print seq
+        return seq
+    
+    def load_container_signature(self, containersignature_file):
+        """Load the PRONOM container-signature file
+        and convert sequences to regular expressions
+        """
+        tree = CET.parse(containersignature_file)
+        # load and have container signatures converted
+        self.sequenceSignature = {}
+        for signature in tree.getroot().findall('ContainerSignatures/ContainerSignature'):
+            signatureId = signature.get('Id')
+            signatureSequence = signature.findall('Files/File/BinarySignatures/InternalSignatureCollection/InternalSignature/ByteSequence/SubSequence')
+            self.sequenceSignature[signatureId] = []
+            for sequence in signatureSequence:
+                self.sequenceSignature[signatureId].append(self.convert_container_sequence(sequence[0].text))
+        # find PUIDs which trigger container matching
+        self.puidTriggers = {}
+        triggers = tree.find('TriggerPuids')
+        for puid in triggers.findall('TriggerPuid'):
+            self.puidTriggers[puid.get('Puid')] = True
+        # map PUID to container signatureId
+        self.puidMapping = {}
+        mappings = tree.find('FileFormatMappings')
+        for mapping in mappings.findall('FileFormatMapping'):
+            if mapping.get('signatureId') not in self.puidMapping:
+                self.puidMapping[mapping.get('signatureId')] = []
+            self.puidMapping[mapping.get('signatureId')].append(mapping.get('Puid'))
+#        print "sequences:\n",self.sequenceSignature
+#        print "trigger:\n",self.puidTriggers
+#        print "mapping:\n",self.puidMapping
+#        exit()
+
+    def load_fido_xml(self, file):
+        """Load the fido format information from @param file.
+           As a side-effect, set self.formats
+           @return list of ElementTree.Element, one for each format.
+        """
+        tree = ET.parse(file)
+        #print "Loaded format specs in {0:>6.2f}ms".format((t1 - t0) * 1000)
+        #TODO: Handle empty regexes properly; perhaps remove from the format list
+        for element in tree.getroot().findall('./format'):
+            puid = self.get_puid(element)
+            # Handle over-writes in multiple file loads
+            existing = self.puid_format_map.get(puid, False) 
+            if  existing:
+                # Already have one, so replace old with new!
+                self.formats[self.formats.index(existing)] = element
+            else:
+                self.formats.append(element)
+            self.puid_format_map[puid] = element
+            # Build some structures to speed things up
+            self.puid_has_priority_over_map[puid] = frozenset([puid_element.text for puid_element in element.findall('has_priority_over')])
+        return self.formats
+
+    # To delete a format: (1) remove from self.formats, (2) remove from puid_format_map, (3) remove from selt.puid_has_priority_over_map
+    def get_signatures(self, format):
+        return format.findall('signature')
+    
+    def has_priority_over(self, format, possibly_inferior):
+        return self.get_puid(possibly_inferior)in self.puid_has_priority_over_map[self.get_puid(format)]
+                
+    def get_puid(self, format):
+        return format.find('puid').text
+    
+    def get_patterns(self, signature):
+        return signature.findall('pattern')
+    
+    def get_pos(self, pat):        
+        return pat.find('position').text
+    
+    def get_regex(self, pat):
+        return pat.find('regex').text
+    
+    def get_extension(self, format):
+        return format.find('extension').text
+    
+    def print_matches(self, fullname, matches, delta_t, matchtype=''):
+        """The default match handler.  Prints out information for each match in the list.
+           @param fullname is name of the file being matched
+           @param matches is a list of (format, signature)
+           @param delta_t is the time taken for the match.
+           @param matchtype is the type of match (signature, containersignature, extension, fail)
+        """
+        class Info:
+            pass
+        obj = Info()
+        obj.count = self.current_count
+        obj.group_size = len(matches)
+        obj.filename = fullname
+        obj.time = int(delta_t * 1000)
+        obj.filesize = self.current_filesize
+        obj.matchtype = matchtype
+        if len(matches) == 0:
+            sys.stdout.write(self.printnomatch % { "info.time" : obj.time, "info.filesize" : obj.filesize, "info.filename" : obj.filename, "info.count"
+            : obj.count, "info.matchtype" : "fail" } )
+        else:
+            i = 0
+            for (f, s) in matches:
+                i += 1
+                obj.group_index = i
+                obj.puid = self.get_puid(f)
+                obj.formatname = f.find('name').text
+                obj.signaturename = s.find('name').text
+                mime = f.find('mime')
+                obj.mimetype = mime.text if mime != None else None
+                version = f.find('version')
+                obj.version = version.text if version != None else None
+                alias = f.find('alias')
+                obj.alias = alias.text if alias != None else None
+                apple_uti = f.find('apple_uid')
+                obj.apple_uti = apple_uti.text if apple_uti != None else None
+                sys.stdout.write(self.printmatch % { "info.time" : obj.time, "info.puid" : obj.puid, "info.formatname" : obj.formatname, "info.signaturename" : obj.signaturename, "info.filesize" : obj.filesize, "info.filename" : obj.filename, "info.mimetype" : obj.mimetype, "info.matchtype" : obj.matchtype, "info.version" : obj.version, "info.alias" : obj.alias, "info.apple_uti" : obj.apple_uti, "info.group_size" : obj.group_size, "info.group_index" : obj.group_index, "info.count" : obj.count })
+        
+    def print_summary(self, secs):
+        """Print summary information on the number of matches and time taken.
+        """
+        count = self.current_count
+        if not self.quiet:
+            rate = (int(round(count / secs)) if secs != 0 else 9999)
+            #print >> sys.stderr, 'FIDO: Processed %6d files in %6.2f msec, %2d files/sec' %  (count, secs * 1000, rate)
+            sys.stderr.write('FIDO: Processed %6d files in %6.2f msec, %2d files/sec\n' %  (count, secs * 1000, rate))
+                    
+    def identify_file(self, filename):
+        """Identify the type of @param filename.  
+           Call self.handle_matches instead of returning a value.
+        """
+        self.current_file = filename
+        self.matchtype = "signature"
+        try:
+            t0 = time.clock()
+            f = open(filename, 'rb')
+            size = os.stat(filename)[6]
+            self.current_filesize = size
+            if self.current_filesize == 0:
+                sys.stderr.write("FIDO: Zero byte file (empty): Path is: {0}\n".format(filename))
+            bofbuffer, eofbuffer = self.get_buffers(f, size, seekable=True)
+            matches = self.match_formats(bofbuffer, eofbuffer)               
+            # from here is also repeated in walk_zip
+            # we should make this uniform in a next version!
+            #
+            # filesize is made conditional because files with 0 bytes
+            # are falsely characterised being 'rtf' (due to wacky sig)
+            # in these cases we try to match the extension instead
+            if len(matches) > 0 and self.current_filesize > 0:
+                self.handle_matches(filename, matches, time.clock() - t0, self.matchtype)
+            elif len(matches) == 0 or self.current_filesize == 0:
+                matches = self.match_extensions(filename)
+                self.handle_matches(filename, matches, time.clock() - t0, "extension")
+            # till here matey!
+            if self.zip:
+                self.identify_contents(filename, type=self.container_type(matches))
+        except IOError:
+            #print >> sys.stderr, "FIDO: Error in identify_file: Path is {0}".format(filename)
+            sys.stderr.write("FIDO: Error in identify_file: Path is {0}\n".format(filename))
+
+    def identify_contents(self, filename, fileobj=None, type=False):
+        """Identify each item in a container (such as a zip or tar file).  Call self.handle_matches on each item.
+           @param fileobj could be a file, or a stream.
+        """
+        if type == False:
+            return
+        elif type == 'zip':
+            self.walk_zip(filename, fileobj)
+        elif type == 'tar':
+            self.walk_tar(filename, fileobj)
+        else: # TODO: ouch!
+            raise RuntimeError("Unknown container type: " + repr(type))
+                
+    def identify_multi_object_stream(self, stream):
+        """Does not work!
+           Stream may contain one or more objects each with an HTTP style header that must include content-length.
+           The headers consist of keyword:value pairs terminated by a newline.  There must be a newline following the headers.
+        """
+        offset = 0
+        while True:
+            t0 = time.clock()
+            content_length = -1
+            for line in stream:
+                offset += len(line)
+                if line == '\n':
+                    if content_length < 0:
+                        raise EnvironmentError("No content-length provided.")
+                    else:
+                        break
+                pair = line.lower().split(':', 2)
+                if pair[0] == 'content-length':
+                    content_length = int(pair[1])
+            if content_length == -1:
+                return
+            # Consume exactly content-length bytes        
+            self.current_file = 'STDIN!(at ' + str(offset) + ' bytes)'
+            self.current_filesize = content_length
+            bofbuffer, eofbuffer = self.get_buffers(stream, content_length)
+            matches = self.match_formats(bofbuffer, eofbuffer)
+            # MdR: this needs attention
+            if len(matches) > 0:
+                self.handle_matches(self.current_file, matches, time.clock() - t0, "signature")
+            elif len(matches) == 0 or self.current_filesize == 0:
+                matches = self.match_extensions(self.current_file)
+                self.handle_matches(self.current_file, matches, time.clock() - t0, "extension")
+                
+    def identify_stream(self, stream, filename):
+        """Identify the type of @param stream.  
+           Call self.handle_matches instead of returning a value.
+           Does not close stream.
+        """
+        t0 = time.clock()
+        bofbuffer, eofbuffer, bytes_read = self.get_buffers(stream, length=None)
+        self.current_filesize = bytes_read
+        self.current_file = 'STDIN'
+        matches = self.match_formats(bofbuffer, eofbuffer)
+        # MdR: this needs attention
+        if len(matches) > 0:
+            self.handle_matches(self.current_file, matches, time.clock() - t0, "signature")
+        elif len(matches) == 0 or self.current_filesize == 0:
+            # we can only determine the filename from the STDIN stream
+            # on Linux, on Windows there is not a (simple) way to do that
+            if (os.name != "nt"):
+                try:
+                    self.current_file = os.readlink("/proc/self/fd/0")
+                except:
+                    if filename is not None:
+                        self.current_file = filename
+                    else:
+                        self.current_file = 'STDIN'
+            else:
+                if filename is not None:
+                    self.current_file = filename
+            matches = self.match_extensions(self.current_file)
+            # we have to reset self.current_file if not on Windows
+            if (os.name != "nt"):
+                self.current_file = 'STDIN'
+            self.handle_matches(self.current_file, matches, time.clock() - t0, "extension")
+                    
+    def container_type(self, matches):
+        """Determine if one of the @param matches is the format of a container that we can look inside of (e.g., zip, tar).
+           @return False, zip, or tar.
+        """
+        for (format, unused) in matches:
+            container = format.find('container')
+            if container != None:
+                return container.text
+        return False
+    
+    def blocking_read(self, file, bytes_to_read):
+        bytes_read = 0
+        buffer = ''
+        while bytes_read < bytes_to_read:
+            readbuffer = file.read(bytes_to_read - bytes_read)
+            buffer += readbuffer
+            bytes_read = len(buffer)
+            # break out if EOF is reached.
+            if readbuffer == '':
+                break
+        return buffer
+                                  
+    def get_buffers(self, stream, length=None, seekable=False):
+        """Return buffers from the beginning and end of stream and the number of bytes read
+           if there may be more bytes in the stream.  
+        
+           If length is None, return the length as found. 
+           If seekable is False, the steam does not support a seek operation.
+        """
+        bytes_to_read = self.bufsize if length == None else min(length, self.bufsize)
+        bofbuffer = self.blocking_read(stream, bytes_to_read)
+        bytes_read = len(bofbuffer)
+        if length == None:
+            # A stream with unknown length; have to keep two buffers around
+            prevbuffer = bofbuffer
+            while True:
+                buffer = self.blocking_read(stream, self.bufsize)
+                bytes_read += len(buffer)
+                if len(buffer) == self.bufsize:
+                    prevbuffer = buffer
+                else:
+                    eofbuffer = prevbuffer if len(buffer) == 0 else prevbuffer[-(self.bufsize - len(buffer)):] + buffer
+                    break
+            return bofbuffer, eofbuffer, bytes_read
+        else:
+            bytes_unread = length - len(bofbuffer)
+            if bytes_unread == 0:
+                eofbuffer = bofbuffer
+            elif bytes_unread < self.bufsize:
+                # The buffs overlap
+                eofbuffer = bofbuffer[bytes_unread:] + self.blocking_read(stream, bytes_unread)
+            elif bytes_unread == self.bufsize:
+                eofbuffer = self.blocking_read(stream, self.bufsize)
+            elif seekable:  # easy case when we can just seek!
+                stream.seek(length - self.bufsize)
+                eofbuffer = self.blocking_read(stream, self.bufsize)
+            else:
+                # We have more to read and know how much.    
+                # n*bufsize + r = length
+                (n, r) = divmod(bytes_unread, self.bufsize)
+                # skip n-1*bufsize bytes
+                for unused_i in xrange(1, n):
+                    self.blocking_read(stream, self.bufsize)
+                # skip r bytes
+                self.blocking_read(stream, r)
+                # and read the remaining bufsize bytes into the eofbuffer
+                eofbuffer = self.blocking_read(stream, self.bufsize)
+            return bofbuffer, eofbuffer
+    
+    def walk_zip(self, filename, fileobj=None):
+        """Identify the type of each item in the zip 
+           @param fileobj.  If fileobj is not provided, open 
+           @param filename.
+           Call self.handle_matches instead of returning a value.
+        """
+        # IN 2.7+: with zipfile.ZipFile((fileobj if fileobj != None else filename), 'r') as stream:
+        import zipfile, tempfile
+        try:
+            zipstream = None
+            zipstream = zipfile.ZipFile((fileobj if fileobj != None else filename), 'r')    
+            for item in zipstream.infolist():
+                if item.file_size == 0:
+                    continue           #TODO: Find a better test for isdir
+                t0 = time.clock()
+                # with zipstream.open(item) as f:
+                f = None
+                try:
+                    f = zipstream.open(item)
+                    item_name = filename + '!' + item.filename
+                    self.current_file = item_name
+                    self.current_filesize = item.file_size
+                    if self.current_filesize == 0:
+                        sys.stderr.write("FIDO: Zero byte file (empty): Path is: {0}\n".format(item_name))
+                    bofbuffer, eofbuffer = self.get_buffers(f, item.file_size)
+                finally:
+                    if f != None: f.close()
+                matches = self.match_formats(bofbuffer, eofbuffer)
+                if len(matches) > 0 and self.current_filesize > 0:
+                    self.handle_matches(item_name, matches, time.clock() - t0, "signature")
+                elif len(matches) == 0 or self.current_filesize == 0:
+                    matches = self.match_extensions(item_name)
+                    self.handle_matches(item_name, matches, time.clock() - t0, "extension")
+                if self.container_type(matches):
+                        target = tempfile.SpooledTemporaryFile(prefix='Fido')
+                        #with zipstream.open(item) as source:
+                        try:
+                            source = zipstream.open(item)
+                            self.copy_stream(source, target)
+                            #target.seek(0)
+                            self.identify_contents(item_name, target, self.container_type(matches))
+                        finally:
+                            source.close()
+        except IOError:
+            sys.stderr.write("FIDO: ZipError {0}\n".format(filename))
+        except zipfile.BadZipfile:
+            sys.stderr.write("FIDO: ZipError {0}\n".format(filename))
+            
+        finally:
+            if zipstream != None: zipstream.close()
+
+    def walk_tar(self, filename, fileobj):
+        """Identify the type of each item in the tar 
+           @param fileobj.  If fileobj is not provided, open 
+           @param filename.
+           Call self.handle_matches instead of returning a value.
+        """
+        import tarfile
+        tarstream = None
+        try:
+            tarstream = tarfile.TarFile(filename, fileobj=fileobj, mode='r')
+            for item in tarstream.getmembers():
+                if item.isfile():
+                    t0 = time.clock()
+                    f = tarstream.extractfile(item)
+                    tar_item_name = filename + '!' + item.name
+                    self.current_file = tar_item_name
+                    self.current_filesize = item.size
+                    bofbuffer, eofbuffer = self.get_buffers(f, item.size)
+                    matches = self.match_formats(bofbuffer, eofbuffer)
+                    self.handle_matches(tar_item_name, matches, time.clock() - t0)
+                    if self.container_type(matches):
+                        f.seek(0)
+                        self.identify_contents(tar_item_name, f, self.container_type(matches))
+                        f.close()
+        except tarfile.TarError:
+                sys.stderr.write("FIDO: Error: TarError {0}\n".format(filename))
+        finally:
+            if tarstream != None: tarstream.close()
+
+    def as_good_as_any(self, f1, match_list):
+        """Return True if the proposed format is as good as any in the match_list.
+           For example, if there is no format in the match_list that has priority over the proposed one"""
+        if match_list != []:
+            f1_puid = self.get_puid(f1)
+            for (f2, unused) in match_list:
+                if f1 == f2:
+                    continue
+                elif f1_puid in self.puid_has_priority_over_map[self.get_puid(f2)]:
+                    return False
+        return True
+    
+    def buffered_read(self, file_pos, overlap):
+        """Buffered read of data chunks
+        """
+        buf = ""
+        if not overlap:
+            bufsize = self.container_bufsize
+        else:
+            bufsize = self.container_bufsize + self.overlap_range
+        file_end = self.current_filesize
+        file_handle = file(self.current_file, 'rb')
+        file_handle.seek(file_pos)
+        if file_end - file_pos < bufsize:
+            file_read = file_end - file_pos
+        else:
+            file_read = self.bufsize
+        buf = file_handle.read(file_read)
+        return buf
+
+    def read_container(self,parent_buffer,parent_result):
+        """Header of compound containers can be further away than default 128 KB buffer 
+           especially with big files containing binary objects.
+           This function reads containers in chunks of 512 KB (defaults['container_bufsize'])
+           Each chunk is inspected with the PRONOM container sequences.
+           Each chunk smuggles in a piece from the previous chunk to prevent 
+           cutting off patterns we are looking for in the middle.
+           This method is somewhat slower than reading the complete file at once.
+           This is to prevent Fido to potentially crash in the midst of scanning a very big file.
+           NOTE (MdR): this piece of code is still a bit quirky
+           as it does not yet takes byte positions into account which
+           are available in the DROID container signature file
+        """
+        container_result = []
+        nobuffer = False
+        overlap = False
+        self.overlap_range = 512 # bytes
+        container_hit = False
+        passes = 1
+        container_buffer = ""
+        # TODO: find better way to handle zip contents
+        # for now: ugly hack, but working
+        # this slows down because the zip is re-opened on each item
+        # if "!" is in filename, it is a zip item
+#        if "!" in self.current_file:
+#            import zipfile, tempfile
+#            zip, item = self.current_file.split("!")
+#            zipitem = tempfile.SpooledTemporaryFile(prefix='Fido')
+            #with zipstream.open(item) as source:
+#            try:
+#                source = zipstream.open(item)
+#                self.copy_stream(source, target)
+#                target.seek(0)
+#                self.identify_contents(item_name, target, self.container_type(matches))
+#            finally:
+#                source.close()
+        #exit()
+        # in case argument 'nocontainer' is set
+        # read default bofbuffer
+        if self.nocontainer or self.current_filesize <= self.bufsize or self.current_file == "STDIN":
+            passes = 1
+            nobuffer = True
+        else:
+            passes = int(float(self.current_filesize / self.container_bufsize) + 1)
+        pos = 0
+        for i in xrange(passes):
+            if nobuffer is True:
+                container_buffer = parent_buffer
+            else:
+                if i == 0:
+                    pos = 0
+                else:
+                    pos = ((self.container_bufsize * i) - self.overlap_range)
+                    overlap = True
+                container_buffer = self.buffered_read(pos, overlap)
+            for (container_id,container_regexes) in self.sequenceSignature.iteritems():
+                # set hitcounter in case a container entry
+                # has more than one regex
+                hitcounter = 0
+                if len(container_regexes) > 0:
+                    for container_regex in container_regexes:
+                        if re.search(container_regex, container_buffer):
+                            hitcounter += 1
+                            # if the hitcounter matches the number of regexes
+                            # then it must be a positive hit, else continue
+                            # to match the rest of the sequences
+                            if hitcounter < len(container_regexes):
+                                continue
+                            self.matchtype = "container"
+                            for container_puid in self.puidMapping[container_id]:
+                                for container_format in self.formats:
+                                    if container_format.find('puid').text == container_puid:
+                                        if self.as_good_as_any(container_format, parent_result):
+                                            for container_sig in self.get_signatures(container_format):
+                                                container_result.append((container_format, container_sig))
+                                            break
+        return container_result
+
+    def match_formats(self, bofbuffer, eofbuffer):
+        """Apply the patterns for formats to the supplied buffers.
+           @return a match list of (format, signature) tuples. 
+           The list has inferior matches removed.
+        """
+        self.current_count += 1
+        #t0 = time.clock()
+        result = []
+        container_result = []
+        for format in self.formats:
+            try:
+                self.current_format = format
+                if self.as_good_as_any(format, result):
+                    for sig in self.get_signatures(format):
+                        self.current_sig = sig
+                        success = True
+                        for pat in self.get_patterns(sig):
+                            self.current_pat = pat
+                            pos = self.get_pos(pat)
+                            regex = self.get_regex(pat)
+                            #print 'trying ', regex
+                            if pos == 'BOF':
+                                if not re.match(regex, bofbuffer):
+                                    success = False
+                                    break
+                            elif pos == 'EOF':
+                                if not re.search(regex, eofbuffer):
+                                    success = False
+                                    break
+                            elif pos == 'VAR':
+                                if not re.search(regex, bofbuffer):
+                                    success = False 
+                                    break
+                            elif pos == 'IFB':
+                                if not re.search(regex, bofbuffer):
+                                    success = False 
+                                    break
+                        if success:
+                            result.append((format, sig))
+                            # check if file needs to be parsed with container signature
+                            # we skip files with extension "zip" (x-fmt/263)
+                            ext = os.path.splitext(self.current_file)[1].lower().lstrip(".")
+                            if format.find('puid').text in self.puidTriggers and ext != "zip":
+                                container_result = self.read_container(bofbuffer,result)
+                                if len(container_result) != 0:
+                                    for (k,v) in container_result:
+                                        result.append((k,v))
+                            break
+            except Exception as e:
+                sys.stderr.write(str(e)+"\n")
+                continue
+            # TODO: MdR: needs some <3
+            #print "Unexpected error:", sys.exc_info()[0], e
+            #sys.stdout.write('***', self.get_puid(format), regex)
+            
+        #        t1 = time.clock()
+        #        if t1 - t0 > 0.02:
+        #            print >> sys.stderr, "FIDO: Slow ID", self.current_file
+        result = [match for match in result if self.as_good_as_any(match[0], result)]
+        result = list(set(result)) # remove duplicate results, this is due to ??? in self.read_container(), needs fix
+        return result
+    
+    def match_extensions(self, filename):
+        "Return the list of (format, self.externalsig) for every format whose extension matches the filename."
+        myext = os.path.splitext(filename)[1].lower().lstrip(".")
+        result = []
+        if len(myext) > 0:
+            for element in self.formats:
+                if element.findall('extension') != None:
+                    for format in element.findall('extension'):
+                        if myext == format.text:
+                            result.append((element, self.externalsig))
+                            break
+        result = [match for match in result if self.as_good_as_any(match[0], result)]
+        return result
+    
+    def copy_stream(self, source, target):
+        while True:
+            buf = source.read(self.bufsize)
+            if len(buf) == 0:
+                break
+            target.write(buf)
+             
+def list_files(roots, recurse=False):
+    "Return the files one at a time.  Roots could be a fileobj or a list."
+    for root in roots:
+        root = (root if root[-1] != '\n' else root[:-1])
+        root = os.path.normpath(root)
+        if os.path.isfile(root):
+            yield root
+        else:
+            for path, unused, files in os.walk(root):
+                for f in files:
+                    yield os.path.join(path, f)
+                if recurse == False:
+                    break
+            
+def main(arglist=None):
+    # The argparse package was introduced in 2.7
+    t0 = time.clock() 
+    from argparselocal import ArgumentParser, RawTextHelpFormatter    
+    if arglist == None:
+        arglist = sys.argv[1:]
+    if len(arglist) == False:
+        arglist.append("-h")        
+    parser = ArgumentParser(description=defaults['description'], epilog=defaults['epilog'], fromfile_prefix_chars='@', formatter_class=RawTextHelpFormatter)
+    parser.add_argument('-v', default=False, action='store_true', help='show version information')
+    parser.add_argument('-q', default=False, action='store_true', help='run (more) quietly')
+    parser.add_argument('-recurse', default=False, action='store_true', help='recurse into subdirectories')
+    parser.add_argument('-zip', default=False, action='store_true', help='recurse into zip and tar files')
+    parser.add_argument('-nocontainer', default=False, action='store_true', help='disable deep scan of container documents, increases speed but may reduce accuracy with big files')
+    parser.add_argument('-pronom_only', default=False, action='store_true', help='disables loading of format extensions file, only PRONOM signatures are loaded, may reduce accuracy of results')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('-input', default=False, help='file containing a list of files to check, one per line. - means stdin')
+    group.add_argument('files', nargs='*', default=[], metavar='FILE', help='files to check. If the file is -, then read content from stdin. In this case, python must be invoked with -u or it may convert the line terminators.')
+    parser.add_argument('-filename', default=None, help='filename if file contents passed through STDIN')
+    parser.add_argument('-useformats', metavar='INCLUDEPUIDS', default=None, help='comma separated string of formats to use in identification')
+    parser.add_argument('-nouseformats', metavar='EXCLUDEPUIDS', default=None, help='comma separated string of formats not to use in identification')
+    parser.add_argument('-matchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use on match. See nomatchprintf, README.txt.')
+    parser.add_argument('-nomatchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use if no match. See README.txt')
+    parser.add_argument('-bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default='+str(defaults['bufsize'])+' bytes)')
+    parser.add_argument('-container_bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default='+str(defaults['container_bufsize'])+' bytes)')
+    
+    parser.add_argument('-loadformats', default=None, metavar='XML1,...,XMLn', help='comma separated string of XML format files to add.')
+    parser.add_argument('-confdir', default=None, help='configuration directory to load_fido_xml, for example, the format specifications from.')
+    
+    # what is this doing here only once?   
+    #mydir = os.path.abspath(os.path.dirname(__file__))
+
+    # PROCESS ARGUMENTS
+    args = parser.parse_args(arglist)
+#    print args
+#    sys.exit()
+    # process confdir
+    # load versions.xml
+    # and stick it in defaults
+    if args.confdir:
+        versionsFile = os.path.join(os.path.abspath(args.confdir), defaults['versions_file'])
+    else:
+        versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versions_file'])
+    try:
+        versions = VET.parse(versionsFile)
+    except Exception, e:
+        sys.stderr.write("An error occured loading versions.xml:\n{0}".format(e))
+        sys.exit()
+    defaults['xml_pronomSignature'] = versions.find("pronomSignature").text
+#    defaults['xml_pronomContainerSignature'] = versions.find("pronomContainerSignature").text
+    defaults['containersignature_file'] = versions.find("pronomContainerSignature").text
+    defaults['xml_fidoExtensionSignature'] = versions.find("fidoExtensionSignature").text
+    defaults['format_files'] = []
+    defaults['format_files'].append(defaults['xml_pronomSignature'])
+    if args.pronom_only:
+        versionHeader = "FIDO v{0} ({1}, {2})\n".format(version,defaults['xml_pronomSignature'],defaults['containersignature_file'])
+    else:
+        versionHeader = "FIDO v{0} ({1}, {2}, {3})\n".format(version,defaults['xml_pronomSignature'],defaults['containersignature_file'],defaults['xml_fidoExtensionSignature'])
+        defaults['format_files'].append(defaults['xml_fidoExtensionSignature'])
+    
+    if args.v :
+        sys.stdout.write(versionHeader)
+        sys.exit(0)
+    if args.matchprintf != None:
+        args.matchprintf = args.matchprintf.decode('string_escape')
+    if args.nomatchprintf != None:
+        args.nomatchprintf = args.nomatchprintf.decode('string_escape')
+    fido = Fido(quiet=args.q, bufsize=args.bufsize, container_bufsize=args.container_bufsize,
+                printmatch=args.matchprintf, printnomatch=args.nomatchprintf, zip=args.zip, nocontainer = args.nocontainer, conf_dir=args.confdir)
+    
+    #TODO: Allow conf options to be dis-included
+    if args.loadformats:
+        for file in args.loadformats.split(','):
+            fido.load_fido_xml(file)
+        
+    #TODO: remove from maps
+    if args.useformats:
+        args.useformats = args.useformats.split(',')
+        fido.formats = [f for f in fido.formats if f.find('puid').text in args.useformats]
+    elif args.nouseformats:
+        args.nouseformats = args.nouseformats.split(',')
+        fido.formats = [f for f in fido.formats if f.find('puid').text not in args.nouseformats]
+ 
+    # Set up to use stdin, or open input files:
+    if args.input == '-':
+        args.files = sys.stdin
+    elif args.input:
+        args.files = open(args.input, 'r')
+    
+    # RUN
+    try:
+        if not args.q:
+            sys.stderr.write(versionHeader)
+            sys.stderr.flush()
+        if (not args.input) and len(args.files) == 1 and args.files[0] == '-':
+            if fido.zip == True:
+                raise RuntimeError("Multiple content read from stdin not yet supported.")
+                sys.exit(1)
+                fido.identify_multi_object_stream(sys.stdin)
+            else:
+                fido.identify_stream(sys.stdin, args.filename)
+        else:
+            for file in list_files(args.files, args.recurse):
+                fido.identify_file(file)
+    except KeyboardInterrupt:
+        msg = "FIDO: Interrupt while identifying file {0}"
+        sys.stderr.write(msg.format(fido.current_file))
+        sys.exit(1)
+        
+    if not args.q:
+        sys.stdout.flush()
+        fido.print_summary(time.clock() - t0)
+        sys.stderr.flush()
+
+if __name__ == '__main__':
+    main()