libis-format 0.9.1

data/data/lias_formats.xml

@@ -0,0 +1,106 @@
+<?xml version="1.0"?>
+<formats xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="0.3" xmlns="fido-formats.xsd">
+  <format>
+    <puid>lias-fmt/101</puid>
+    <mime>text/xml</mime>
+    <name>Extensible Markup Language</name>
+    <version>1.0</version>
+    <alias>XML (1.0)</alias>
+    <pronom_id>638</pronom_id>
+    <extension>xml</extension>
+    <has_priority_over>fmt/101</has_priority_over>
+    <signature>
+      <name>XML 1.0</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A.{0,3}\x3c\?xml\s+version\s+=\s+(?:"|')1\.0(?:"|')</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/001</puid>
+    <name>Textura TS print file</name>
+    <mime>text/plain</mime>
+    <signature>
+      <name>Textura TS print file</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A(\f#%\*P66610\r\nBACKUP-NAME : \(PR\)[A-Z/\d]*/TS\d* ON USER\r\nCREATION-DATE : \d{6}\r\n\*%#\f )?\f {6}(\r\n){2}={75}   (\r\n){2}L I J S T   V A N   D E   U I T K E R I N G E N   S Y N D\.   P R E M I E S {21}EUR {4}(\r\n){2}={75}   (\r\n){2}CENTR[\S ]*\d\d/\d\d/\d\d   (\r\n){2}={75}   \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)(\r\n){2}NUMMER DRAGER   TYPE DRAGER   SCHRIJFCODE   DICHTHEID   BLOCKINGSFACTOR \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {41}AANTAL {17}BEDRAG  \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}POSTASSIGNATIES :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}CIRCULAIRE CHEQUES :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}CIRC\.CHEQUES \(TERUG ORGAN\.\) :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}OVERSCHRIJVINGEN : </regex>
+      </pattern>
+      <!--
+      -->
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}ALGEMEEN TOTAAL :  </regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\r\nHANDTEKENING\(EN\) AFGEVER: {9}( .){10}(\r\n){3}HANDTEKENING VOOR ONTVANGST: {6}( .){10}\r\n( \f|\f\x1a)\x00+\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/002</puid>
+    <name>Textura VP/S print file</name>
+    <mime>text/plain</mime>
+    <has_priority_over>lias-fmt/001</has_priority_over>
+    <signature>
+      <name>Textura VP/S print file</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A(\f#%\*P66610\r\nBACKUP-NAME : \(PR\)[A-Z/\d]*V[PS]\d* ON USER\r\nCREATION-DATE : \d{6}\r\n\*%#\f )?\f {49}(LIJST|LISTE) AVP .{55}(REEKS|SERIE) :[ \d]*\r\n {48}-{57} {3}\r\n\r\n</regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\r\n( \f|\f\x1a)\x00+\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/189.word</puid>
+    <name>Microsoft Office Open XML - Word</name>
+    <extension>docx</extension>
+    <has_priority_over>x-fmt/263</has_priority_over>
+    <has_priority_over>fmt/189</has_priority_over>
+    <has_priority_over>fido-fmt/189.word</has_priority_over>
+    <signature>
+      <name>Microsoft Office Open XML - Word</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\APK\x03\x04</regex>
+      </pattern>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A.{30}\[Content_Types\]\.xml \xa2</regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\x00\x00word/.{1,20}\.xmlPK\x01\x02\x2d.{0,4000}\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+</formats>

data/data/types.yml

@@ -0,0 +1,213 @@
+---
+# This lists all the types the converters know about along with the mime types and file extensions.
+# The first file extension in the list is the default one that will be used when a file of that type is created.
+# The mime types need to be unique. Some mime types need to be invented like for instance for PDF/A. The MimeType
+# class should take care of that.
+# Preferably the file extensions are unique too. If not, the first matching entry in the list will be used when a
+# reverse lookup from extension to type identifier is performed. However, file extensions will typically not be used
+# to determine type identifier or mime types. So you should be fairly safe when the file extensions are not unique.
+
+IMAGE:
+  TIFF:
+    NAME: Tagged Image File Format (TIFF)
+    MIME: image/tiff
+    EXTENSIONS: tif,tiff
+
+  JPEG2000:
+    NAME: Joint Photographic Experts Group 2000 (JPEG 2000)
+    MIME: image/jp2
+    EXTENSIONS: jp2
+
+  JPEG:
+    NAME: Joint Photographic Experts Group (JPEG)
+    MIME: image/jpeg
+    EXTENSIONS: jpg,jpe,jpeg
+
+  PNG:
+    NAME: Portable Network Graphics (PNG)
+    MIME: image/png
+    EXTENSIONS: png
+
+  BMP:
+    NAME: Device Independent Bitmap (DIP/BMP)
+    MIME: image/bmp,image/x-ms-bmp
+    EXTENSIONS: bmp
+
+  GIF:
+    NAME: Graphics Interchange Format (GIF)
+    MIME: image/gif
+    EXTENSIONS: gif
+
+  PBM:
+    NAME: Portable Bitmap Format (PBM)
+    PUID: fmt/409
+    MIME: image/x‑portable‑bitmap
+    EXTENSIONS: pbm
+
+  PGM:
+    NAME: Portable GrayMap Format (PGM)
+    PUID: fmt/406
+    MIME: image/x‑portable‑graymap
+    EXTENSIONS: pgm
+
+  PPM:
+    NAME: Portable Pixel Map (PPM)
+    PUID: fmt/408
+    MIME: image/x‑portable‑pixmap
+    EXTENSIONS: ppm
+
+AUDIO:
+
+  WAV:
+    NAME: Waveform Audio File Format (WAVE)
+    MIME: audio/x-wav
+    EXTENSIONS: wav
+
+  MP3:
+    NAME: MPEG-1 or MPEG-2 Audio Layer III (MP3)
+    MIME: audio/mpeg
+    EXTENSIONS: mp3
+
+  FLAC:
+    NAME: Free Lossless Audio Codec (FLAC)
+    MIME: audio/flac
+    EXTENSIONS: flac
+
+VIDEO:
+
+  MPEG:
+    NAME: Moving Picture Experts Group (MPEG-1/MPEG-2)
+    MIME: video/mpeg
+    EXTENSIONS: mpg mpeg mp1 mp2 mp3 m1v m1a m2a mpa mpv
+
+  MPEG4:
+    NAME: Moving Picture Experts Group (MPEG-4)
+    MIME: video/mp4
+    EXTENSIONS: mp4,mpeg4
+
+  MJP2:
+    NAME: Motion JPEG 2000 (MJP2)
+    MIME: video/mj2
+    EXTENSIONS: mj2, mjp2
+
+  QTFF:
+    NAME: QuickTime File Format (QTFF)
+    MIME: video/quicktime
+    EXTENSIONS: mov, qt
+
+  AVI:
+    NAME: Audio Video Interleave (AVI)
+    MIME: video/x-msvideo
+    EXTENSIONS: avi
+
+  OGGV:
+    NAME: OGG Video (OGGV)
+    MIME: video/ogg
+    EXTENSIONS: ogv
+
+  WMV:
+    NAME: Windows Media Video (WMV)
+    MIME: video/x-ms-wmv
+    EXTENSIONS: wmv
+
+  DV:
+    NAME: Digital Video (DV)
+    MIME: video/dv
+    EXTENSIONS: dv
+
+  FLASH:
+    NAME: Flash video (FLV)
+    MIME: video/x-flv
+    EXTENSIONS: flv
+
+DOCUMENT:
+
+  TXT:
+    NAME: Unformatted text
+    MIME: text/plain
+    EXTENSIONS: txt
+
+  RTF:
+    NAME: Rich Text Format (RTF)
+    PUID: fmt/45
+    MIME: text/rtf application/rtf
+    EXTENSIONS: rtf
+
+  HTML:
+    NAME: HyperText Markup Language (HTML)
+    MIME: text/html
+    EXTENSIONS: html, htm
+
+  MSDOC:
+    NAME: Microsoft Word Document (DOC)
+    PUID: fmt/609 fmt/39 x-fmt/2 x-fmt/129 x-fmt/273 x-fmt/274 x-fmt/275 x-fmt/276 fmt/37 fmt/38 fmt/40
+    MIME: application/vnd.ms-word application/msword
+    EXTENSIONS: doc
+
+  MSDOCX:
+    NAME: Microsoft Word OpenXML Document (DOCX)
+    PUID: fmt/412 fido-fmt/189.word lias-fmt/189.word
+    MIME: application/vnd.openxmlformats-officedocument.wordprocessingml.document
+    EXTENSIONS: docx
+
+  MSXLS:
+    NAME: Microsoft Excel Spreadsheet (XLS)
+    MIME: application/vnd.ms-excel,application/msexcel
+    EXTENSIONS: xls
+
+  MSXLSX:
+    NAME: Microsoft Excel OpenXML Spreadheet (XSLX)
+    PUID: fido-fmt/189.xl
+    MIME: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    EXTENSIONS: xslx
+
+  MSPPT:
+    NAME: Microsoft Powerpoint Presentation (PPT)
+    MIME: application/vnd.ms-powerpoint,application/mspowerpoint
+    EXTENSIONS: ppt
+
+  MSPPTX:
+    NAME: Microsoft Powerpoint OpenXML Presentation (PPTX)
+    PUID: fido-fmt/189.ppt
+    MIME: application/vnd.openxmlformats-officedocument.presentationml.presentation
+    EXTENSIONS: pptx
+
+  PDF:
+    NAME: Adobe Portable Document Format (PDF)
+    PUID: fmt/14 fmt/15 fmt/16 fmt/17 fmt/18 fmt/19 fmt/20 fmt/276
+    MIME: application/pdf
+    EXTENSIONS: pdf
+
+  PDFA:
+    NAME: Adobe Portable Document Format for Archives (PDFA)
+    PUID: fmt/95 fmt/354 fmt/476 fmt/477 fmt/478 fmt/479 fmt/480 fmt/481
+    MIME: application/pdf
+    EXTENSIONS: pdf
+
+  WORDPERFECT:
+    NAME: WordPerfect Document (WPD)
+    PUID: x-fmt/44 x-fmt/394
+    MIME: application/vnd.wordperfect
+    EXTENSIONS: wpd
+
+  XML:
+    NAME: Extensible Markup Language (XML)
+    MIME: text/xml
+    PUID: fmt/101
+    EXTENSIONS: xml
+
+  SHAREPOINT_MAP:
+    NAME: Sharepoint mapping file
+    # This is again an invented mime type. Its actually an XML ...
+    MIME: text/xml/sharepoint_map
+    PUID: fmt/101
+    EXTENSIONS: xml
+
+ARCHIVE:
+
+  EAD:
+    NAME: Encoded Archival Description (EAD)
+    # This is again an invented mime type. Its actually an XML ...
+    MIME: archive/ead
+    PUID: fmt/101
+    EXTENSIONS: ead,xml

data/lib/libis/format/converter/base.rb

@@ -0,0 +1,103 @@
+# coding: utf-8
+
+### require 'tools/string'
+
+require 'libis/tools/logger'
+require 'libis/format/type_database'
+
+require_relative 'repository'
+
+module Libis
+  module Format
+    module Converter
+
+      class Base
+        include Libis::Tools::Logger
+
+        def input_types
+          raise RuntimeError, 'Method #input_types needs to be overridden in converter'
+        end
+
+        protected
+
+        def output_types
+          raise RuntimeError, 'Method #output_types needs to be overridden in converter'
+        end
+
+        attr_accessor :source, :options, :flags
+
+        def init(_)
+          raise RuntimeError, 'Method #init should be implemented in converter'
+        end
+
+        def do_convert(_, _)
+          raise RuntimeError, 'Method #do_convert should be implemented in converter'
+        end
+
+        public
+
+        def initialize( source = nil, options = {}, flags = {} )
+          @source = source
+          @options = options ? options : {}
+          @flags = flags ? flags : {}
+          init(source.to_s rescue nil)
+        end
+
+        def convert(target, format = nil)
+          do_convert(target, format)
+        end
+
+        def Base.inherited( klass )
+
+          Repository.register klass
+
+          class << self
+
+            def conversions
+              input_types.inject({}) do |input_type, hash|
+                hash[input_type] = output_types
+                hash
+              end
+            end
+
+            def input_type?(type_id)
+              input_types.include? type_id
+            end
+
+            def output_type?(type_id)
+              output_types.include? type_id
+            end
+
+            def input_mimetype?(mimetype)
+              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              input_type? type_id
+            end
+
+            def output_mimetype?(mimetype)
+              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              output_type? type_id
+            end
+
+            def conversion?(input_type, output_type)
+              conversions[input_type] and conversions[input_type].any? { |t| t == output_type }
+            end
+
+            def output_for(input_type)
+              conversions[input_type]
+            end
+
+            def extension?(extension)
+              !TypeDatabase.ext_types(extension).first.nil?
+            end
+
+          end
+
+
+        end
+
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/chain.rb

@@ -0,0 +1,80 @@
+# coding: utf-8
+
+require 'fileutils'
+
+require 'libis/tools/logger'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+    module Converter
+
+      class Chain
+        include ::Libis::Tools::Logger
+
+        def initialize(converter_chain)
+          @converter_chain = converter_chain
+        end
+
+        def to_array
+          @converter_chain
+        end
+
+        def convert(src_file, target_file, operations = [])
+
+          chain = @converter_chain.clone
+
+          my_operations = {}
+
+          # sanity check: check if the required operations are supported by at least one converter in the chain
+          operations.each do |k,v|
+            method = k.to_s.downcase.to_sym
+            chain_element = @converter_chain.reverse.detect { |c| c[:converter].new.respond_to? method }
+            if chain_element
+              my_operations[chain_element[:converter]] ||= {}
+              my_operations[chain_element[:converter]][method] = v
+            else
+              error "No converter in the converter chain supports '#{method.to_s}'. Continuing conversion without this operation."
+            end
+          end
+
+          temp_files = []
+
+          # noinspection RubyParenthesesAroundConditionInspection
+          while (chain_element = chain.shift)
+
+            target_type = chain_element[:target]
+            converter_class = chain_element[:converter]
+            converter = converter_class.new(src_file)
+
+            my_operations[converter_class].each do |k,v|
+              converter.send k, v
+            end
+
+            target = target_file
+
+            unless chain.empty?
+              target += '.temp.' + TypeDatabase.instance.type2ext(target_type)
+              target += '.' + TypeDatabase.instance.type2ext(target_type) while File.exist? target
+              temp_files << target
+            end
+
+            FileUtils.mkdir_p File.dirname(target)
+
+            converter.convert(target, target_type)
+
+            src_file = target
+
+          end
+
+          temp_files.each do |f|
+            File.delete(f)
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/repository.rb

@@ -0,0 +1,110 @@
+# coding: utf-8
+
+require 'set'
+require 'singleton'
+
+require 'libis/tools/logger'
+
+require_relative 'chain'
+
+module Libis
+  module Format
+    module Converter
+
+      class Repository
+        include Singleton
+        include ::Libis::Tools::Logger
+
+        attr_reader :converters
+        attr_writer :converters_glob
+
+        def initialize
+          @converters = Set.new
+          @converters_glob = File.join(File.basename(__FILE__), '*_converter.rb')
+        end
+
+        def Repository.register(converter_class)
+          instance.converters.add? converter_class
+        end
+
+        def Repository.get_converters
+          if instance.converters.empty?
+            Dir.glob(instance.converters_glob).each do |filename|
+              # noinspection RubyResolve
+              require File.expand_path(filename)
+            end
+          end
+          instance.converters
+        end
+
+        def Repository.get_converter_chain(src_type, tgt_type, operations = [])
+          msg = "conversion from #{src_type.to_s} to #{tgt_type.to_s}"
+          chain_list = recursive_chain src_type, tgt_type, operations
+          if chain_list.length > 1
+            warn "Found more than one conversion chain for #{msg}. Picking the first one."
+          end
+          if chain_list.empty?
+            error "No conversion chain found for #{msg}"
+            return nil
+          end
+          chain_list.each do |chain|
+            msg = "Base chain: #{src_type.to_s}"
+            chain.each do |node|
+              msg += "->#{node[:converter].name}:#{node[:target].to_s}"
+            end
+            debug msg
+          end
+          ::Libis::Format::Converters::Chain.new(chain_list[0])
+        end
+
+        private
+
+        def Repository.recursive_chain(src_type, tgt_type, operations, chains_found = [], current_chain = [])
+          return chains_found unless current_chain.length < 8 # upper limit of converter chain we want to consider
+
+          get_converters.each do |converter|
+            if converter.conversion? src_type, tgt_type and !current_chain.any? { |c|
+              c[:converter] == converter and c[:target] == tgt_type }
+              node = Hash.new
+              node[:converter] = converter
+              node[:target] = tgt_type
+              sequence = current_chain.dup
+              sequence << node
+              # check if the chain supports all the operations
+              success = true
+              operations.each do |op, _|
+                success = false unless sequence.any? do |n|
+                  n[:converter].new.respond_to? op.to_s.downcase.to_sym
+                end
+              end
+              if success
+                # we only want to remember the shortest converter chains
+                if !chains_found.empty? and sequence.length < chains_found[0].length
+                  chains_found.clear
+                end
+                chains_found << sequence if chains_found.empty? or sequence.length == chains_found[0].length
+              end
+            end
+          end
+
+          return chains_found unless chains_found.empty? or current_chain.length + 1 < chains_found[0].length
+
+          get_converters.each do |converter|
+            next unless converter.input_type? src_type
+            converter.output_types(src_type).each do |tmp_type|
+              # would like to enable the following for optimalization, but some operation may require such a step
+              # next if tmp_type == src_type
+              # next if current_chain.any? { |c| c[:target] == tmp_type}
+              recursive_chain(tmp_type, tgt_type, operations, chains_found,
+                              current_chain.dup << {:converter => converter, :target => tmp_type})
+            end
+          end
+
+          chains_found
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter.rb

@@ -0,0 +1,11 @@
+# code utf-8
+
+module Libis
+  module Format
+    module Converter
+      autoload :Base, 'libis/format/converter/base'
+      autoload :Chain, 'libis/format/converter/chain'
+      autoload :Repository, 'libis/format/converter/repository'
+    end
+  end
+end

data/lib/libis/format/droid.rb

@@ -0,0 +1,38 @@
+require 'os'
+require 'tempfile'
+require 'csv'
+require 'singleton'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+module Libis
+  module Format
+
+    class Droid
+      include ::Libis::Tools::Logger
+      include Singleton
+
+      def self.run(file)
+        instance.run file
+      end
+
+      def run(file)
+        droid_dir = File.join(File.dirname(__FILE__), '..','..','..','tools','droid')
+        droid_cmd = File.join(droid_dir, OS.windows? ? 'droid.bat' : 'droid.sh')
+        profile = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .profile', nil)
+        report = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .csv', nil)
+        result = Libis::Tools::Command.run droid_cmd, '-a', file.escape_for_string, '-p', profile, '-q'
+        warn "DROID profile errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        result = Libis::Tools::Command.run droid_cmd, '-e', report, '-p', profile, '-q'
+        warn "DROID report errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        File.delete profile
+        result = CSV.read(report , headers: true, header_converters: [:downcase, :symbol])
+        File.delete report
+        result.map{|r|r.to_hash}
+      end
+    end
+
+  end
+end