libis-format 0.9.5-java

data/data/lias_formats.xml

@@ -0,0 +1,106 @@
+<?xml version="1.0"?>
+<formats xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="0.3" xmlns="fido-formats.xsd">
+  <format>
+    <puid>lias-fmt/101</puid>
+    <mime>text/xml</mime>
+    <name>Extensible Markup Language</name>
+    <version>1.0</version>
+    <alias>XML (1.0)</alias>
+    <pronom_id>638</pronom_id>
+    <extension>xml</extension>
+    <has_priority_over>fmt/101</has_priority_over>
+    <signature>
+      <name>XML 1.0</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A.{0,3}\x3c\?xml\s+version\s+=\s+(?:"|')1\.0(?:"|')</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/001</puid>
+    <name>Textura TS print file</name>
+    <mime>text/plain</mime>
+    <signature>
+      <name>Textura TS print file</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A(\f#%\*P66610\r\nBACKUP-NAME : \(PR\)[A-Z/\d]*/TS\d* ON USER\r\nCREATION-DATE : \d{6}\r\n\*%#\f )?\f {6}(\r\n){2}={75}   (\r\n){2}L I J S T   V A N   D E   U I T K E R I N G E N   S Y N D\.   P R E M I E S {21}EUR {4}(\r\n){2}={75}   (\r\n){2}CENTR[\S ]*\d\d/\d\d/\d\d   (\r\n){2}={75}   \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)(\r\n){2}NUMMER DRAGER   TYPE DRAGER   SCHRIJFCODE   DICHTHEID   BLOCKINGSFACTOR \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {41}AANTAL {17}BEDRAG  \r\n</regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}POSTASSIGNATIES :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}CIRCULAIRE CHEQUES :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}CIRC\.CHEQUES \(TERUG ORGAN\.\) :  </regex>
+      </pattern>
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}OVERSCHRIJVINGEN : </regex>
+      </pattern>
+      <!--
+      -->
+      <pattern>
+        <position>VAR</position>
+        <regex>(?s)\r\n {5}ALGEMEEN TOTAAL :  </regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\r\nHANDTEKENING\(EN\) AFGEVER: {9}( .){10}(\r\n){3}HANDTEKENING VOOR ONTVANGST: {6}( .){10}\r\n( \f|\f\x1a)\x00+\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/002</puid>
+    <name>Textura VP/S print file</name>
+    <mime>text/plain</mime>
+    <has_priority_over>lias-fmt/001</has_priority_over>
+    <signature>
+      <name>Textura VP/S print file</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A(\f#%\*P66610\r\nBACKUP-NAME : \(PR\)[A-Z/\d]*V[PS]\d* ON USER\r\nCREATION-DATE : \d{6}\r\n\*%#\f )?\f {49}(LIJST|LISTE) AVP .{55}(REEKS|SERIE) :[ \d]*\r\n {48}-{57} {3}\r\n\r\n</regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\r\n( \f|\f\x1a)\x00+\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+  <format>
+    <puid>lias-fmt/189.word</puid>
+    <name>Microsoft Office Open XML - Word</name>
+    <extension>docx</extension>
+    <has_priority_over>x-fmt/263</has_priority_over>
+    <has_priority_over>fmt/189</has_priority_over>
+    <has_priority_over>fido-fmt/189.word</has_priority_over>
+    <signature>
+      <name>Microsoft Office Open XML - Word</name>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\APK\x03\x04</regex>
+      </pattern>
+      <pattern>
+        <position>BOF</position>
+        <regex>(?s)\A.{30}\[Content_Types\]\.xml \xa2</regex>
+      </pattern>
+      <pattern>
+        <position>EOF</position>
+        <regex>(?s)\x00\x00word/.{1,20}\.xmlPK\x01\x02\x2d.{0,4000}\Z</regex>
+      </pattern>
+    </signature>
+  </format>
+</formats>

data/data/types.yml

@@ -0,0 +1,217 @@
+---
+# This lists all the types the converters know about along with the mime types and file extensions.
+# The first file extension in the list is the default one that will be used when a file of that type is created.
+# The mime types need to be unique. Some mime types need to be invented like for instance for PDF/A. The MimeType
+# class should take care of that.
+# Preferably the file extensions are unique too. If not, the first matching entry in the list will be used when a
+# reverse lookup from extension to type identifier is performed. However, file extensions will typically not be used
+# to determine type identifier or mime types. So you should be fairly safe when the file extensions are not unique.
+
+IMAGE:
+  TIFF:
+    NAME: Tagged Image File Format (TIFF)
+    MIME: image/tiff
+    EXTENSIONS: tif,tiff
+
+  JP2:
+    NAME: Joint Photographic Experts Group 2000 (JPEG 2000)
+    MIME: image/jp2
+    EXTENSIONS: jp2
+
+  JPG:
+    NAME: Joint Photographic Experts Group (JPEG)
+    MIME: image/jpeg
+    EXTENSIONS: jpg,jpe,jpeg
+
+  PNG:
+    NAME: Portable Network Graphics (PNG)
+    MIME: image/png
+    EXTENSIONS: png
+
+  BMP:
+    NAME: Device Independent Bitmap (DIP/BMP)
+    MIME: image/bmp,image/x-ms-bmp
+    EXTENSIONS: bmp
+
+  GIF:
+    NAME: Graphics Interchange Format (GIF)
+    MIME: image/gif
+    EXTENSIONS: gif
+
+  PBM:
+    NAME: Portable Bitmap Format (PBM)
+    PUID: fmt/409
+    MIME: image/x‑portable‑bitmap
+    EXTENSIONS: pbm
+
+  PGM:
+    NAME: Portable GrayMap Format (PGM)
+    PUID: fmt/406
+    MIME: image/x‑portable‑graymap
+    EXTENSIONS: pgm
+
+  PPM:
+    NAME: Portable Pixel Map (PPM)
+    PUID: fmt/408
+    MIME: image/x‑portable‑pixmap
+    EXTENSIONS: ppm
+
+AUDIO:
+
+  WAV:
+    NAME: Waveform Audio File Format (WAVE)
+    MIME: audio/x-wav
+    EXTENSIONS: wav
+
+  MP3:
+    NAME: MPEG-1 or MPEG-2 Audio Layer III (MP3)
+    MIME: audio/mpeg
+    EXTENSIONS: mp3
+
+  FLAC:
+    NAME: Free Lossless Audio Codec (FLAC)
+    MIME: audio/flac
+    EXTENSIONS: flac
+
+VIDEO:
+
+  MPEG:
+    NAME: Moving Picture Experts Group (MPEG-1/MPEG-2)
+    MIME: video/mpeg
+    EXTENSIONS: mpg mpeg mp1 mp2 mp3 m1v m1a m2a mpa mpv
+
+  MPEG4:
+    NAME: Moving Picture Experts Group (MPEG-4)
+    MIME: video/mp4
+    EXTENSIONS: mp4,mpeg4
+
+  MJP2:
+    NAME: Motion JPEG 2000 (MJP2)
+    MIME: video/mj2
+    EXTENSIONS: mj2, mjp2
+
+  QTFF:
+    NAME: QuickTime File Format (QTFF)
+    MIME: video/quicktime
+    EXTENSIONS: mov, qt
+
+  AVI:
+    NAME: Audio Video Interleave (AVI)
+    MIME: video/x-msvideo
+    EXTENSIONS: avi
+
+  OGGV:
+    NAME: OGG Video (OGGV)
+    MIME: video/ogg
+    EXTENSIONS: ogv
+
+  WMV:
+    NAME: Windows Media Video (WMV)
+    MIME: video/x-ms-wmv
+    EXTENSIONS: wmv
+
+  DV:
+    NAME: Digital Video (DV)
+    MIME: video/dv
+    EXTENSIONS: dv
+
+  FLASH:
+    NAME: Flash video (FLV)
+    MIME: video/x-flv
+    EXTENSIONS: flv
+
+TEXT:
+
+  TXT:
+    NAME: Unformatted text
+    MIME: text/plain
+    EXTENSIONS: txt
+
+  RTF:
+    NAME: Rich Text Format (RTF)
+    PUID: fmt/45
+    MIME: text/rtf application/rtf
+    EXTENSIONS: rtf
+
+  HTML:
+    NAME: HyperText Markup Language (HTML)
+    MIME: text/html
+    EXTENSIONS: html, htm
+
+  MSDOC:
+    NAME: Microsoft Word Document (DOC)
+    PUID: fmt/609 fmt/39 x-fmt/2 x-fmt/129 x-fmt/273 x-fmt/274 x-fmt/275 x-fmt/276 fmt/37 fmt/38 fmt/40
+    MIME: application/vnd.ms-word application/msword
+    EXTENSIONS: doc
+
+  MSDOCX:
+    NAME: Microsoft Word OpenXML Document (DOCX)
+    PUID: fmt/412 fido-fmt/189.word lias-fmt/189.word
+    MIME: application/vnd.openxmlformats-officedocument.wordprocessingml.document
+    EXTENSIONS: docx
+
+  PDF:
+    NAME: Adobe Portable Document Format (PDF)
+    PUID: fmt/14 fmt/15 fmt/16 fmt/17 fmt/18 fmt/19 fmt/20 fmt/276
+    MIME: application/pdf
+    EXTENSIONS: pdf
+
+  PDFA:
+    NAME: Adobe Portable Document Format for Archives (PDFA)
+    PUID: fmt/95 fmt/354 fmt/476 fmt/477 fmt/478 fmt/479 fmt/480 fmt/481
+    MIME: application/pdf
+    EXTENSIONS: pdf
+
+  WORDPERFECT:
+    NAME: WordPerfect Document (WPD)
+    PUID: x-fmt/44 x-fmt/394
+    MIME: application/vnd.wordperfect
+    EXTENSIONS: wpd
+
+  XML:
+    NAME: Extensible Markup Language (XML)
+    MIME: text/xml
+    PUID: fmt/101
+    EXTENSIONS: xml
+
+  SHAREPOINT_MAP:
+    NAME: Sharepoint mapping file
+    # This is again an invented mime type. Its actually an XML ...
+    MIME: text/xml/sharepoint_map
+    PUID: fmt/101
+    EXTENSIONS: xml
+
+TABULAR:
+
+  MSXLS:
+    NAME: Microsoft Excel Spreadsheet (XLS)
+    MIME: application/vnd.ms-excel,application/msexcel
+    EXTENSIONS: xls
+
+  MSXLSX:
+    NAME: Microsoft Excel OpenXML Spreadheet (XSLX)
+    PUID: fido-fmt/189.xl
+    MIME: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    EXTENSIONS: xslx
+
+PRESENTATION:
+
+  MSPPT:
+    NAME: Microsoft Powerpoint Presentation (PPT)
+    MIME: application/vnd.ms-powerpoint,application/mspowerpoint
+    EXTENSIONS: ppt
+
+  MSPPTX:
+    NAME: Microsoft Powerpoint OpenXML Presentation (PPTX)
+    PUID: fido-fmt/189.ppt
+    MIME: application/vnd.openxmlformats-officedocument.presentationml.presentation
+    EXTENSIONS: pptx
+
+ARCHIVE:
+
+  EAD:
+    NAME: Encoded Archival Description (EAD)
+    # This is again an invented mime type. Its actually an XML ...
+    MIME: archive/ead
+    PUID: fmt/101
+    EXTENSIONS: ead,xml

data/lib/libis/format/config.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require 'os'
+
+module Libis
+  module Format
+
+    # noinspection RubyConstantNamingConvention
+    Config = ::Libis::Tools::Config
+
+    Config[:converter_chain_max_level] = 8
+
+    Config[:java_path] = 'java'
+    Config[:soffice_path] = 'soffice'
+    Config[:ghostscript_path] = 'gs'
+    # Config[:pdfa_path] =
+    #     File.absolute_path(
+    #         File.join(
+    #             File.dirname(__FILE__), '..', '..', '..', 'tools', 'pdf', 'pdfa', 'pdfa'
+    #         )
+    #     )
+    Config[:droid_path] =
+        File.absolute_path(
+            File.join(
+                File.dirname(__FILE__), '..', '..', '..', 'tools', 'droid', OS.windows? ? 'droid.bat' : 'droid.sh'
+            )
+        )
+    Config[:fido_path] =
+        File.absolute_path(
+            File.join(
+                File.dirname(__FILE__), '..', '..', '..', 'tools', 'fido', OS.windows? ? 'fido.bat' : 'fido.sh'
+            )
+        )
+
+  end
+end

data/lib/libis/format/converter/base.rb

@@ -0,0 +1,101 @@
+# coding: utf-8
+
+### require 'tools/string'
+require 'tmpdir'
+require 'libis/tools/logger'
+require 'libis/format/type_database'
+
+require_relative 'repository'
+
+module Libis
+  module Format
+    module Converter
+
+      class Base
+        include Libis::Tools::Logger
+
+        attr_reader :options, :flags
+
+        def initialize
+          @options = {}
+          @flags = {}
+        end
+
+        def convert(source, target, format, opts = {})
+          unless File.exist? source
+            error "Cannot find file '#{source}'."
+            return nil
+          end
+          @options.merge!(opts[:options]) if opts[:options]
+          @flags.merge!(opts[:flags]) if opts[:flags]
+        end
+
+        def self.input_types(_ = nil)
+          raise RuntimeError, 'Method #input_types needs to be overridden in converter'
+        end
+
+        def self.output_types(_ = nil)
+          raise RuntimeError, 'Method #output_types needs to be overridden in converter'
+        end
+
+
+        def using_temp(target)
+          tempfile = File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname(['convert', File.extname(target)], File.basename(target, '.*').gsub(/\s/, '_')))
+          result = yield tempfile
+          return nil unless result
+          FileUtils.move result, target
+          target
+        end
+
+        def Base.inherited( klass )
+
+          Repository.register klass
+
+          class << self
+
+            def conversions
+              input_types.inject({}) do |hash, input_type|
+                hash[input_type] = output_types
+                hash
+              end
+            end
+
+            def input_type?(type_id)
+              input_types.include? type_id
+            end
+
+            def output_type?(type_id)
+              output_types.include? type_id
+            end
+
+            def input_mimetype?(mimetype)
+              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              input_type? type_id
+            end
+
+            def output_mimetype?(mimetype)
+              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              output_type? type_id
+            end
+
+            def conversion?(input_type, output_type)
+              conversions[input_type] and conversions[input_type].any? { |t| t == output_type }
+            end
+
+            def output_for(input_type)
+              conversions[input_type]
+            end
+
+            def extension?(extension)
+              !TypeDatabase.ext_types(extension).first.nil?
+            end
+
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/chain.rb

@@ -0,0 +1,167 @@
+# coding: utf-8
+
+require 'fileutils'
+require 'deep_dive'
+
+require 'libis/tools/logger'
+require 'libis/tools/extend/hash'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+    module Converter
+
+      class Chain
+        include ::Libis::Tools::Logger
+        include DeepDive
+
+        def initialize(source_format, target_format, operations = {})
+          @source_format = source_format.to_sym
+          @target_format = target_format.to_sym
+          @operations = operations || {}
+          @converter_chain = []
+        end
+
+        # @param [Libis::Format::Converter::Base.class] converter
+        # @return [Array[Hash]]
+        def append(converter)
+          return [] unless converter
+          valid_chain_nodes(converter).map do |node|
+            self.ddup.add_chain_node(node)
+          end.compact
+        end
+
+        def closed?
+          !@converter_chain.empty? &&
+              @converter_chain.first[:input].to_sym == @source_format &&
+              @converter_chain.last[:output].to_sym == @target_format
+        end
+
+        def valid?
+          closed? && apply_operations
+        end
+
+        def to_array
+          @converter_chain
+        end
+
+        def size
+          @converter_chain.size
+        end
+
+        alias_method :length, :size
+
+        def to_s
+          "#{@source_format}->-#{@converter_chain.map do |node|
+            "#{node[:converter].name.gsub(/^.*::/, '')}#{node[:operations].empty? ? '' :
+                "(#{node[:operations].each do |operation|
+                  "#{operation[:method]}:#{operation[:argument]}"
+                end.join(',')})"}->-#{node[:output]}"
+          end.join('->-')}"
+        end
+
+        def convert(src_file, target_file)
+
+          unless valid?
+            error 'Converter chain is not valid'
+            return nil
+          end
+
+          temp_files = []
+
+          # noinspection RubyParenthesesAroundConditionInspection
+          result = @converter_chain.each_with_index do |node, i|
+
+            target_type = node[:output]
+            converter_class = node[:converter]
+            converter = converter_class.new
+
+            node[:operations].each do |operation|
+              converter.send operation[:method], operation[:argument]
+            end if node[:operations]
+
+            target = target_file
+
+            if i < size - 1
+              target += ".temp.#{TypeDatabase.type_extentions(target_type).first}"
+              target += ".#{TypeDatabase.type_extentions(target_type).first}" while File.exist? target
+              temp_files << target
+            end
+
+            FileUtils.mkdir_p File.dirname(target)
+
+            src_file = converter.convert(src_file, target, target_type)
+
+            break :failed unless src_file
+
+          end
+
+          temp_files.each do |f|
+            FileUtils.rm(f, force: true)
+          end
+
+          result == :failed ? nil : target_file
+
+        end
+
+        def valid_chain_nodes(converter)
+          source_format = @converter_chain.last[:output] rescue @source_format
+          nodes = []
+          if converter.input_types.include? source_format
+            converter.output_types(source_format).each do |format|
+              node = {converter: converter, input: source_format, output: format}
+              next if node_exists?(node)
+              nodes << node
+            end
+          end
+          nodes
+        end
+
+        def add_chain_node(node = {})
+          last_converter = @converter_chain.last
+          source_format = last_converter ? last_converter[:output] : @source_format
+          node[:input] ||= source_format
+          return nil unless node[:input] == source_format
+          return nil unless node[:output] && node[:converter].output_types(source_format).include?(node[:output])
+          return nil unless node[:converter].input_types.include? source_format
+          return nil if node_exists?(node)
+          @converter_chain << node
+          # debug "Chain: #{self}"
+          self
+        end
+
+        def apply_operations
+          temp_chain = @converter_chain.reverse.ddup
+          applied = true
+          operations = @operations && @operations.ddup || {}
+          while (operation = operations.shift)
+            method = operation.first.to_s.to_sym
+            applied &&= :found == temp_chain.each do |node|
+              next unless node[:converter].instance_methods.include?(method)
+              node[:operations] ||= []
+              node[:operations] << {method: method, argument: operation.last}
+              break :found
+            end
+          end
+          if applied && operations.empty?
+            @converter_chain = temp_chain.reverse
+            @operations.clear
+            return true
+          end
+          false
+        end
+
+
+        private
+
+        def node_exists?(node)
+          @converter_chain.detect do |n|
+            n[:converter] == node[:converter] && n[:input] == node[:input] && n[:output] == node[:output]
+          end
+        end
+
+      end
+
+    end
+  end
+end