libis-format 1.2.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bb468afbfdd7031a466736c6d355919395bf295356032f6b3d3488908bb61b5
-  data.tar.gz: b2ddc4588f0f3a1ac4ddde120ce726af21ab9013c98bae303674dca2033fec58
+  metadata.gz: f563ad694fd3d0c9e4a9f0da3309c99ac4b103e6027fcc9cda1606f0f67549c2
+  data.tar.gz: 5cee3f4b3bb1e51db492ee14bf919b0ce0fa69e267292380364c32b4eec40b4e
 SHA512:
-  metadata.gz: 176685c4eb8fac9734666928c7d22b18cc870158b4a45151b35f5cc81922d4f6f0521af026719efb04d9b3c3920ae887c271739b91ade104e5d1f2e6651e8f3e
-  data.tar.gz: f3ca3c913aef08d75ff1dadef3790e7ad6838429bc91dbcff3173391f8d052d3f23ef9bcf3309c4c062650064947821cc046431cfe1196644964c04236cff6a7
+  metadata.gz: 409ea1fac8f8be4d2519f189090207b6a550a1b1ce174a71fa1886c69b667001459d4bdccd850dd55373199ae5553ebd02183e15eebc88a73d0ac3f8093f7d12
+  data.tar.gz: d3e4d77ddcaab1f5050aaf4acb02e74a221566abedb94062254efbc24d18703c8fdd739644b1cd5435b54a4497e77c82bdcbcb218d121bfaf135e483662aca06

data/.gitignore

@@ -18,4 +18,3 @@ mkmf.log
 *.pyc
 tools/pdf/
 /test*
-.rake_tasks~

data/.travis.yml

@@ -1,8 +1,7 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
-
+sudo: false
 rvm:
   - 2.3
   - 2.4
@@ -10,47 +9,37 @@ rvm:
   - 2.6
   - ruby-head
   - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
   - openjdk10
   - openjdk11
 matrix:
   exclude:
     - rvm: 2.3
       jdk: openjdk10
-    - rvm: 2.3
-      jdk: openjdk11
     - rvm: 2.4
       jdk: openjdk10
-    - rvm: 2.4
-      jdk: openjdk11
     - rvm: 2.5
       jdk: openjdk10
-    - rvm: 2.5
-      jdk: openjdk11
     - rvm: 2.6
       jdk: openjdk10
-    - rvm: 2.6
-      jdk: openjdk11
     - rvm: ruby-head
       jdk: openjdk10
-    - rvm: ruby-head
-      jdk: openjdk11
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
-
-env:
-  - LC_ALL=en_IE.UTF-8
-
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
   - sudo apt-get update -y -q
-  - sudo apt-get install -y -q ffmpeg gstreamer0.10-ffmpeg
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
   - sudo pip install --upgrade pip
   - sudo pip install --upgrade --ignore-installed opf-fido six
   - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
@@ -58,8 +47,15 @@ before_install:
   - sudo unzip droid.zip -d /opt/droid
   - sudo chmod 755 /opt/droid/droid.sh
   - sudo locale-gen "en_US.UTF-8"
-  - sudo dpkg-reconfigure locales
-  - echo "8\n9\n" | sudo dpkg-reconfigure tzdata
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
+env:
+  - LC_ALL=en_US.UTF-8
+  - LANG=en_US.UTF-8
 addons:
   apt:
     packages:

data/Gemfile

@@ -3,5 +3,3 @@ source 'https://rubygems.org'
 gem 'coveralls', group: :test, require: false
 
 gemspec name: 'libis-format', development_group: :test
-
-gem 'standard'

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -27,7 +27,9 @@ RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgra
  && rm fop.tar.gz
 
 RUN echo "8\n9\n" | apt-get install tzdata
-ADD ./tools /tools
-ADD ./data /config
-ADD ./spec/data /data
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
 CMD /bin/bash

data/base/rework_path

@@ -1,25 +1,20 @@
 #!/bin/bash
 
-#file=${1/$SRC_PATH/$TGT_PATH}
-#echo "$1 -> $file"
-
 CMD=$1
 shift
 
 readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
 
 args=("$@")
-echo "args: ${args[@]}"
-#args="${args[@]/$SRC_PATH/$TGT_PATH}"
-#args="${args[@]/$SRC_TMP/$TGT_TMP}"
+#echo "args: ${args[@]}"
 for map in "${mapping[@]}"
 do
         map="${map//[$'\t\r\n ']}"
         readarray -d '=' -t m <<< "$map"
         src="${m[0]//[$'\t\r\n ']}"
         tgt="${m[1]//[$'\t\r\n ']}"
-        echo "$src -> $tgt"
-        args="${args[@]/$src/$tgt}"
-        echo "args: ${args[@]}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
 done
-echo $CMD "${args[@]}"
+${CMD} "${args[@]}"

data/data/PDFA_def.ps

@@ -5,10 +5,10 @@
 % unless the user modifies the corresponding line below.
 
 % Define entries in the document Info dictionary :
-/ICCProfile ([**ICC profile**])
+/ICCProfile ([** Fill in ICC profile location **])
 def
 
-[ [**METADATA**]
+[ /Title (Title)
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -35,6 +35,6 @@ def
   /Type /OutputIntent             % Must be so (the standard requires).
   /S /GTS_PDFA1                   % Must be so (the standard requires).
   /DestOutputProfile {icc_PDFA}            % Must be so (see above).
-  /OutputConditionIdentifier ([**ICC reference**])
+  /OutputConditionIdentifier ([** Fill in ICC reference name **])
 >> /PUT pdfmark
 [{Catalog} <</OutputIntents [ {OutputIntent_PDFA} ]>> /PUT pdfmark

data/data/types.yml

@@ -96,14 +96,14 @@ AUDIO:
 
   AAC:
     NAME: Advanced Audio Coding (AAC)
-    MIME: audio/aac audio/x-hx-aac-adts
+    MIME: audio/aac
     EXTENSIONS: aac
 
   WMA:
     NAME: Windows Media Audio (WMA)
     MIME: audio/x-ms-wma
     PUID: fmt/132
-    EXTENSIONS: wma asf
+    EXTENSIONS: wma
 
   ALAC:
     NAME: Apple Lossless Audio Codec (ALAC)
@@ -194,7 +194,7 @@ VIDEO:
     NAME: Windows Media Video (WMV)
     MIME: video/x-ms-wmv
     PUID: fmt/133 fmt/441
-    EXTENSIONS: wmv asf
+    EXTENSIONS: wmv
 
   DV:
     NAME: Digital Video (DV)
@@ -319,12 +319,6 @@ TEXT:
 
 TABULAR:
 
-  CSV:
-    NAME: Comma Separated Values
-    MIME: text/csv
-    PUID: x-fmt/18
-    EXTENSIONS: csv
-
   MSXLS:
     NAME: Microsoft Excel Spreadsheet (XLS)
     MIME: application/vnd.ms-excel,application/msexcel
@@ -357,15 +351,8 @@ TABULAR:
 
 PRESENTATION:
 
-  MSPPS:
-    NAME: Microsoft Powerpoint Presentation Show
-    PUID: x-fmt/87
-    MIME: application/vnd.ms-powerpoint
-    EXTENSIONS: pps
-
   MSPPT:
     NAME: Microsoft Powerpoint Presentation (PPT)
-    PUID: fmt/181 fmt/179 fmt/1747 fmt/1748 x-fmt/88 fmt/125 fmt/126
     MIME: application/vnd.ms-powerpoint,application/mspowerpoint
     EXTENSIONS: ppt
 

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -21,10 +22,10 @@ module Libis
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
-    Config[:email2pdf_jar] = File.join(Libis::Format::TOOL_DIR, 'emailconverter.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
@@ -35,7 +36,6 @@ module Libis
         fop: 5 * 60,
         identification_tool: 5 * 60,
         office_to_pdf: 5 * 60,
-        email2pdf: 5 * 60,
         pdf_copy: 5 * 60,
         pdf_merge: 5 * 60,
         pdf_optimizer: 5 * 60,

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,46 +69,13 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
-
-          else
-
-            convert_file(source, target)
-
-          end
-
-        end
+          convert_file(source, target)
 
-        def assemble_and_convert(sources, target)
-          result = {}
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            result = convert_file(f.to_path, target)
-          end
-          result
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)
@@ -154,7 +121,6 @@ module Libis
 
         def convert_file(source, target)
           opts = {global: [], input: [], filter: [], output: []}
-
           opts[:global] << '-hide_banner'
           opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
           opts[:output] << '-vn' # disable input video stream in case it exists
@@ -168,8 +134,10 @@ module Libis
           opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
           opts[:output] << '-ac' << @options[:channels] if @options[:channels]
           opts[:output] << '-f' << @options[:format] if @options[:format]
-
-          Libis::Format::Tool::FFMpeg.run(source, target, opts)
+          result = Libis::Format::Tool::FFMpeg.run(source, target, opts)
+          info "FFMpeg output: #{result}"
+          result
+          target
         end
 
       end

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,7 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +22,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +64,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +86,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +104,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end

data/lib/libis/format/converter/chain.rb

@@ -5,7 +5,7 @@ require 'deep_dive'
 
 require 'libis/tools/logger'
 require 'libis/tools/extend/hash'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 module Libis
   module Format
@@ -68,18 +68,14 @@ module Libis
           end
 
           temp_files = []
-          xtra_files = []
-
-          result = { commands: [] }
 
           # noinspection RubyParenthesesAroundConditionInspection
-          conversion_success = @converter_chain.each_with_index do |node, i|
+          result = @converter_chain.each_with_index do |node, i|
 
             target_type = node[:output]
             converter_class = node[:converter]
             converter = converter_class.new
 
-
             node[:operations].each do |operation|
               converter.send operation[:method], operation[:argument]
             end if node[:operations]
@@ -87,32 +83,24 @@ module Libis
             target = target_file
 
             if i < size - 1
-              target += ".temp.#{TypeDatabase.type_extentions(target_type).first}"
-              target += ".#{TypeDatabase.type_extentions(target_type).first}" while File.exist? target
+              target += ".temp.#{Libis::Format::Library.get_field(target_type, :extensions).first}"
+              target += ".#{Libis::Format::Library.get_field(target_type, :extensions).first}" while File.exist? target
               temp_files << target
             end
 
             FileUtils.mkdir_p File.dirname(target)
 
-            r = converter.convert(src_file, target, target_type)
+            src_file = converter.convert(src_file, target, target_type)
 
-            src_file = r[:files].first
-            xtra_files += r[:files][1..]
             break :failed unless src_file
 
-            result[:commands] << r.merge(converter: converter_class.name)
-
-            :success
-
           end
 
-          result[:files] = [src_file] + xtra_files
-
           temp_files.each do |f|
             FileUtils.rm(f, force: true)
           end
 
-          conversion_success == :failed ? nil : result
+          result == :failed ? nil : target_file
 
         end
 

data/lib/libis/format/converter/fop_pdf_converter.rb

@@ -31,6 +31,8 @@ module Libis
 
           Libis::Format::Tool::FopPdf.run(source, target)
 
+          target
+
         end
 
       end

data/lib/libis/format/converter/image_assembler.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+require_relative 'base'
+require 'libis/format/identifier'
+
+require 'mini_magick'
+
+MiniMagick.logger.level = ::Logger::UNKNOWN
+
+MiniMagick.configure do |config|
+  # config.cli = :graphicsmagick
+  config.validate_on_create = false
+  config.validate_on_write = false
+  config.whiny = false
+end
+
+module Libis
+  module Format
+    module Converter
+
+      # noinspection RubyTooManyInstanceVariablesInspection,DuplicatedCode
+      class ImageAssembler < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format) if format
+          [:PDF, :TIFF, :GIF, :PBM, :PGM, :PPM]
+        end
+
+        def self.category
+          :assembler
+        end
+
+        def image_assemble(_)
+          #force usage of this converter
+        end
+
+        def quiet(v)
+          @quiet = !!v
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+            assemble(source, target, format)
+          elsif File.directory?(source)
+            source_list = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble(source_list, target, format)
+          else
+            image = MiniMagick::Image.open(source) {|b| b.quiet}
+            if image.pages.size > 1
+              assemble(image.pages.map {|page| page.path}, target, format)
+            else
+              assemble([source], target, format)
+            end
+          end
+
+          target
+
+        end
+
+        private
+
+        def assemble(sources, target, format)
+          MiniMagick::Tool::Convert.new do |b|
+            sources.each {|source| b << source}
+            convert.format(format)
+            b << target
+          end
+        end
+
+      end
+
+    end
+  end
+end