libis-format 1.0.7 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09143d3be9aca7d01b5c6ba3512e4b106d4ee2f0e370fad9c565bf0724c61846'
-  data.tar.gz: 5c9f5fb67527e828a2b293ffe6e4bf8616d7a08f67780b963e5f292bb50ae385
+  metadata.gz: 43d209bde1a85973e528730bb3ec4c9cffc58db02f916936fa2144d4e91b81f7
+  data.tar.gz: 04315d80c7e20b6302f9d6d10906f4ec7d5b6046a47902fbeff95ec97bf7732a
 SHA512:
-  metadata.gz: cbf99230e3566a2aacdfd02e97f2622cd83d32601cb772fec50a94c04054754e367ea1a09276e7cdee6d9bc564f8642a63967159a2d3290769c368945fe3c233
-  data.tar.gz: c0f33d0d7d0208ea7ed8678da52dea0dd30ee54438c321a4be0a1b5ccc01f2451591251615775074639b03e4548d9a97cf4218e4ae0b8ce0aa2819e70f12a964
+  metadata.gz: d581e3ac51fc290ba57e27d2bf6bfe5dcfa8f269a8dbdfa656954bff2af7efd8d963177ec1cb309db481bd7b84603df20e8b34bb1800f7c88d93e91e31722532
+  data.tar.gz: 867208ce1766cb4bdec0efc648c7b5a1e1dd9883ffd5818f82324ea3a783b21e50cb81fafdbc04bad1c1da75adb75503c63d9be3936baeeac436a5a92ac9a791

data/.gitignore

@@ -18,3 +18,5 @@ mkmf.log
 *.pyc
 tools/pdf/
 /test*
+/*~
+/*.pdf

data/.travis.yml

@@ -1,41 +1,45 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
+sudo: false
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
   - sudo apt-get update -y -q
-  - sudo apt-get install -y -q ffmpeg gstreamer0.10-ffmpeg
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
   - sudo pip install --upgrade pip
   - sudo pip install --upgrade --ignore-installed opf-fido six
   - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
@@ -43,8 +47,12 @@ before_install:
   - sudo unzip droid.zip -d /opt/droid
   - sudo chmod 755 /opt/droid/droid.sh
   - sudo locale-gen "en_US.UTF-8"
-  - sudo dpkg-reconfigure locales
-  - echo "8\n9\n" | sudo dpkg-reconfigure tzdata
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
 env:
   - LC_ALL=en_US.UTF-8
   - LANG=en_US.UTF-8

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -27,7 +27,9 @@ RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgra
  && rm fop.tar.gz
 
 RUN echo "8\n9\n" | apt-get install tzdata
-ADD ./tools /tools
-ADD ./data /config
-ADD ./spec/data /data
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
 CMD /bin/bash

data/base/rework_path

@@ -1,25 +1,20 @@
 #!/bin/bash
 
-#file=${1/$SRC_PATH/$TGT_PATH}
-#echo "$1 -> $file"
-
 CMD=$1
 shift
 
 readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
 
 args=("$@")
-echo "args: ${args[@]}"
-#args="${args[@]/$SRC_PATH/$TGT_PATH}"
-#args="${args[@]/$SRC_TMP/$TGT_TMP}"
+#echo "args: ${args[@]}"
 for map in "${mapping[@]}"
 do
         map="${map//[$'\t\r\n ']}"
         readarray -d '=' -t m <<< "$map"
         src="${m[0]//[$'\t\r\n ']}"
         tgt="${m[1]//[$'\t\r\n ']}"
-        echo "$src -> $tgt"
-        args="${args[@]/$src/$tgt}"
-        echo "args: ${args[@]}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
 done
-echo $CMD "${args[@]}"
+${CMD} "${args[@]}"

data/lib/libis/format.rb

@@ -3,8 +3,11 @@ require 'libis/format/version'
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
-    autoload :TypeDatabaseImpl, 'libis/format/type_database_impl'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'
@@ -15,4 +18,10 @@ module Libis
     TOOL_DIR = File.join(ROOT_DIR, 'tools')
 
   end
-end
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? !!empty? : !self
+  end unless defined? :blank?
+end

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -23,23 +24,26 @@ module Libis
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
         droid: 20 * 60,
-        ffmpeg: 5 * 60,
+        ffmpeg: 500 * 60,
         fido: 20 * 60,
-        file_tool: 5 * 60,
-        fop: 5 * 60,
-        identification_tool: 5 * 60,
-        office_to_pdf: 5 * 60,
-        pdf_copy: 5 * 60,
-        pdf_merge: 5 * 60,
-        pdf_optimizer: 5 * 60,
-        pdf_split: 5 * 60,
-        pdf_to_pdfa: 5 * 60,
-        pdfa_validator: 5 * 60,
+        file_tool: 20 * 60,
+        fop: 50 * 60,
+        identification_tool: 50 * 60,
+        office_to_pdf: 50 * 60,
+        pdf_select: 50 * 60,
+        pdf_copy: 50 * 60,
+        pdf_merge: 50 * 60,
+        pdf_optimizer: 50 * 60,
+        pdf_split: 50 * 60,
+        pdf_watermark: 50 * 60,
+        pdf_to_pdfa: 50 * 60,
+        pdfa_validator: 50 * 60,
     }
 
   end

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,47 +69,13 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
+          convert_file(source, target)
 
-          else
-
-            convert_file(source, target)
-
-          end
-
-          target
-
-        end
-
-        def assemble_and_convert(sources, target)
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            target = convert_file(f.to_path, target)
-          end
-          target
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,8 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/tools/extend/string'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +23,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +65,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +87,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +105,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end

data/lib/libis/format/converter/chain.rb

@@ -5,7 +5,7 @@ require 'deep_dive'
 
 require 'libis/tools/logger'
 require 'libis/tools/extend/hash'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 module Libis
   module Format
@@ -83,8 +83,8 @@ module Libis
             target = target_file
 
             if i < size - 1
-              target += ".temp.#{TypeDatabase.type_extentions(target_type).first}"
-              target += ".#{TypeDatabase.type_extentions(target_type).first}" while File.exist? target
+              target += ".temp.#{Libis::Format::Library.get_field(target_type, :extensions).first}"
+              target += ".#{Libis::Format::Library.get_field(target_type, :extensions).first}" while File.exist? target
               temp_files << target
             end
 

data/lib/libis/format/converter/image_assembler.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+require_relative 'base'
+require 'libis/format/identifier'
+
+require 'mini_magick'
+
+MiniMagick.logger.level = ::Logger::UNKNOWN
+
+MiniMagick.configure do |config|
+  # config.cli = :graphicsmagick
+  config.validate_on_create = false
+  config.validate_on_write = false
+  config.whiny = false
+end
+
+module Libis
+  module Format
+    module Converter
+
+      # noinspection RubyTooManyInstanceVariablesInspection,DuplicatedCode
+      class ImageAssembler < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format) if format
+          [:PDF, :TIFF, :GIF, :PBM, :PGM, :PPM]
+        end
+
+        def self.category
+          :assembler
+        end
+
+        def image_assemble(_)
+          #force usage of this converter
+        end
+
+        def quiet(v)
+          @quiet = !!v
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+            assemble(source, target, format)
+          elsif File.directory?(source)
+            source_list = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble(source_list, target, format)
+          else
+            image = MiniMagick::Image.open(source) {|b| b.quiet}
+            if image.pages.size > 1
+              assemble(image.pages.map {|page| page.path}, target, format)
+            else
+              assemble([source], target, format)
+            end
+          end
+
+          target
+
+        end
+
+        private
+
+        def assemble(sources, target, format)
+          MiniMagick::Tool::Convert.new do |b|
+            sources.each {|source| b << source}
+            convert.format(format)
+            b << target
+          end
+        end
+
+      end
+
+    end
+  end
+end