libis-format 1.0.7 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09143d3be9aca7d01b5c6ba3512e4b106d4ee2f0e370fad9c565bf0724c61846'
-  data.tar.gz: 5c9f5fb67527e828a2b293ffe6e4bf8616d7a08f67780b963e5f292bb50ae385
+  metadata.gz: f563ad694fd3d0c9e4a9f0da3309c99ac4b103e6027fcc9cda1606f0f67549c2
+  data.tar.gz: 5cee3f4b3bb1e51db492ee14bf919b0ce0fa69e267292380364c32b4eec40b4e
 SHA512:
-  metadata.gz: cbf99230e3566a2aacdfd02e97f2622cd83d32601cb772fec50a94c04054754e367ea1a09276e7cdee6d9bc564f8642a63967159a2d3290769c368945fe3c233
-  data.tar.gz: c0f33d0d7d0208ea7ed8678da52dea0dd30ee54438c321a4be0a1b5ccc01f2451591251615775074639b03e4548d9a97cf4218e4ae0b8ce0aa2819e70f12a964
+  metadata.gz: 409ea1fac8f8be4d2519f189090207b6a550a1b1ce174a71fa1886c69b667001459d4bdccd850dd55373199ae5553ebd02183e15eebc88a73d0ac3f8093f7d12
+  data.tar.gz: d3e4d77ddcaab1f5050aaf4acb02e74a221566abedb94062254efbc24d18703c8fdd739644b1cd5435b54a4497e77c82bdcbcb218d121bfaf135e483662aca06

data/.travis.yml

@@ -1,41 +1,45 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
+sudo: false
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
   - sudo apt-get update -y -q
-  - sudo apt-get install -y -q ffmpeg gstreamer0.10-ffmpeg
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
   - sudo pip install --upgrade pip
   - sudo pip install --upgrade --ignore-installed opf-fido six
   - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
@@ -43,8 +47,12 @@ before_install:
   - sudo unzip droid.zip -d /opt/droid
   - sudo chmod 755 /opt/droid/droid.sh
   - sudo locale-gen "en_US.UTF-8"
-  - sudo dpkg-reconfigure locales
-  - echo "8\n9\n" | sudo dpkg-reconfigure tzdata
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
 env:
   - LC_ALL=en_US.UTF-8
   - LANG=en_US.UTF-8

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -27,7 +27,9 @@ RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgra
  && rm fop.tar.gz
 
 RUN echo "8\n9\n" | apt-get install tzdata
-ADD ./tools /tools
-ADD ./data /config
-ADD ./spec/data /data
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
 CMD /bin/bash

data/base/rework_path

@@ -1,25 +1,20 @@
 #!/bin/bash
 
-#file=${1/$SRC_PATH/$TGT_PATH}
-#echo "$1 -> $file"
-
 CMD=$1
 shift
 
 readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
 
 args=("$@")
-echo "args: ${args[@]}"
-#args="${args[@]/$SRC_PATH/$TGT_PATH}"
-#args="${args[@]/$SRC_TMP/$TGT_TMP}"
+#echo "args: ${args[@]}"
 for map in "${mapping[@]}"
 do
         map="${map//[$'\t\r\n ']}"
         readarray -d '=' -t m <<< "$map"
         src="${m[0]//[$'\t\r\n ']}"
         tgt="${m[1]//[$'\t\r\n ']}"
-        echo "$src -> $tgt"
-        args="${args[@]/$src/$tgt}"
-        echo "args: ${args[@]}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
 done
-echo $CMD "${args[@]}"
+${CMD} "${args[@]}"

data/lib/libis/format.rb

@@ -3,8 +3,11 @@ require 'libis/format/version'
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
-    autoload :TypeDatabaseImpl, 'libis/format/type_database_impl'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -23,7 +24,8 @@ module Libis
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,47 +69,13 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
+          convert_file(source, target)
 
-          else
-
-            convert_file(source, target)
-
-          end
-
-          target
-
-        end
-
-        def assemble_and_convert(sources, target)
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            target = convert_file(f.to_path, target)
-          end
-          target
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,7 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +22,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +64,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +86,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +104,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end

data/lib/libis/format/converter/chain.rb

@@ -5,7 +5,7 @@ require 'deep_dive'
 
 require 'libis/tools/logger'
 require 'libis/tools/extend/hash'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 module Libis
   module Format
@@ -83,8 +83,8 @@ module Libis
             target = target_file
 
             if i < size - 1
-              target += ".temp.#{TypeDatabase.type_extentions(target_type).first}"
-              target += ".#{TypeDatabase.type_extentions(target_type).first}" while File.exist? target
+              target += ".temp.#{Libis::Format::Library.get_field(target_type, :extensions).first}"
+              target += ".#{Libis::Format::Library.get_field(target_type, :extensions).first}" while File.exist? target
               temp_files << target
             end
 

data/lib/libis/format/converter/image_assembler.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+require_relative 'base'
+require 'libis/format/identifier'
+
+require 'mini_magick'
+
+MiniMagick.logger.level = ::Logger::UNKNOWN
+
+MiniMagick.configure do |config|
+  # config.cli = :graphicsmagick
+  config.validate_on_create = false
+  config.validate_on_write = false
+  config.whiny = false
+end
+
+module Libis
+  module Format
+    module Converter
+
+      # noinspection RubyTooManyInstanceVariablesInspection,DuplicatedCode
+      class ImageAssembler < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format) if format
+          [:PDF, :TIFF, :GIF, :PBM, :PGM, :PPM]
+        end
+
+        def self.category
+          :assembler
+        end
+
+        def image_assemble(_)
+          #force usage of this converter
+        end
+
+        def quiet(v)
+          @quiet = !!v
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+            assemble(source, target, format)
+          elsif File.directory?(source)
+            source_list = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble(source_list, target, format)
+          else
+            image = MiniMagick::Image.open(source) {|b| b.quiet}
+            if image.pages.size > 1
+              assemble(image.pages.map {|page| page.path}, target, format)
+            else
+              assemble([source], target, format)
+            end
+          end
+
+          target
+
+        end
+
+        private
+
+        def assemble(sources, target, format)
+          MiniMagick::Tool::Convert.new do |b|
+            sources.each {|source| b << source}
+            convert.format(format)
+            b << target
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/image_converter.rb

@@ -4,7 +4,6 @@ require_relative 'base'
 require 'libis/format/identifier'
 
 require 'mini_magick'
-require 'fileutils'
 
 MiniMagick.logger.level = ::Logger::UNKNOWN
 
@@ -23,11 +22,11 @@ module Libis
       class ImageConverter < Libis::Format::Converter::Base
 
         def self.input_types
-          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2, :PBM, :PGM, :PPM]
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless input_types.include?(format) if format
           [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
         end
 
@@ -46,109 +45,57 @@ module Libis
 
         def quiet(v)
           @quiet = !!v
+          self
         end
 
         def page(nr)
           @page = nr
+          self
         end
 
         def scale(percent)
           @options[:scale] = percent
+          self
         end
 
         def resize(geometry)
           @options[:resize] = geometry
+          self
         end
 
         def quality(value)
           @options[:quality] = value
+          self
         end
 
         def dpi(value)
           @options[:density] = value
+          self
         end
 
         def resample(value)
           @options[:resample] = value
+          self
         end
 
         def flatten(value = true)
-          @options[:flatten] = !!value
+          @options[:flatten] = value
+          self
         end
 
         def colorspace(value)
           @options[:colorspace] = value
+          self
         end
 
         def delete_date(value = true)
-          @delete_date = !!value
+          @delete_date = value
+          self
         end
 
         def profile(icc)
           @profile = icc
-        end
-
-        # Create or use a watermark image.
-        #
-        # The watermark options are:
-        #     - file: watermark image to use
-        #     - text: text to create a watermark from
-        #     - rotation: rotation of the watermark text (counter clockwise in degrees; integer number) - default 30
-        #     - tiles: number of tiles of the watermark - default 4
-        #       - 0: no tiling, so only 1 watermark will be placed with the original size
-        #       - 1: 1 tile, so the watermark will be scaled up to fill the image
-        #       - n > 1: minimum n tiles in both directions
-        #       - n < 0: tile without scaling the watermark
-        #     - size: same as tiles - for backwards compatibility
-        #     - resize: fraction 0.0 - 1.0
-        #     - gap: size of the gap between watermark instances. Fractions as percentage of widht/height. - default 0.2
-        #     - opacity: opacity of the watermark (fraction 0.0 - 1.0) - default 0.1
-        #     - gravity: center point of the overlay - default 'center'
-        # If both options are given, the file will be used as-is if it exists and is a valid image file. Otherwise the
-        # file will be created or overwritten with a newly created watermark image.
-        #
-        # The created watermark file will be a PNG image with transparent background containing the supplied text
-        # slanted by 30 degrees counter-clockwise.
-        #
-        # @param [Hash] options Hash of options for watermark creation.
-        def watermark(options = {})
-          text = options[:text] || '© LIBIS'
-          @wm_tiles = (options[:tiles] || '4').to_i
-          @wm_tiles ||= (options[:size] || '4').to_i
-          @wm_resize = ((options[:resize]).to_f * 100).to_i if options[:resize]
-          @wm_opacity = ((options[:opacity] || 0.1).to_f * 100).to_i
-          @wm_composition = options[:composition] || 'modulate'
-          @wm_gravity = options[:gravity] || 'center'
-          @wm_gap = ((options[:gap] || 0.2).to_f * 100).to_i
-          rotation = 360 - (options[:rotation] || 30).to_i
-          @wm_image = MiniMagick::Image.new(options[:file]) if options[:file]
-          unless @wm_image && @wm_image.valid?
-            image = options[:file] || (Dir::Tmpname.create(%w(wm_image .png)) {|_|})
-            # noinspection RubyResolve
-            MiniMagick::Tool::Convert.new do |convert|
-              # noinspection RubyLiteralArrayInspection
-              convert.quiet
-              convert.background 'transparent'
-              convert.size('2000x2000')
-              convert.gravity 'Center'
-              convert.font('Helvetica').fill('black').pointsize(72) #.stroke('black').strokewidth(1)
-              convert << "label:#{text}"
-              convert.rotate rotation
-              convert.trim.repage.+
-              convert << image
-            end
-            if options[:file]
-              @wm_image = MiniMagick::Image.new(image)
-            else
-              @wm_image = MiniMagick::Image.open(image)
-              File.delete(image)
-            end
-            # noinspection RubyResolve
-            unless @wm_image.valid?
-              error "Problem creating watermark image '#{image}'."
-              @wm_image = nil
-            end
-          end
+          self
         end
 
         def convert(source, target, format, opts = {})
@@ -156,90 +103,25 @@ module Libis
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target, format)
-
-          elsif File.directory?(source)
-            source_list = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-
-            assemble_and_convert(source_list, target, format)
-
-          else
-
-            image = MiniMagick::Image.open(source) { |b| b.quiet }
-
-            if image.pages.size > 1
-              if @page
-                convert_image(image.pages[@page].path, target, format)
-              else
-                # noinspection RubyBlockToMethodReference
-                assemble_and_convert(image.pages.map {|page| page.path}, target, format)
-              end
-            else
-              convert_image(source, target, format)
-            end
-          end
+          convert_image(source, target, format)
 
           target
 
         end
 
-        def assemble_and_convert(sources, target, format)
-
-          warn 'Received multiple images as input and single page format as target.' unless self.class.multipage?(format)
-          converted_pages = sources.inject([]) do |list, path|
-            # noinspection RubyArgCount
-            converted = Tempfile.new(['page-', ".#{Libis::Format::TypeDatabase.type_extentions(format).first}"])
-            convert_image(path, converted.path, format)
-            list << converted
-          end
-          MiniMagick::Tool::Convert.new do |b|
-            b.append unless self.class.multipage?(format)
-            converted_pages.each {|page| b << page.path}
-            b << target
-          end
-          converted_pages.each do |temp_file|
-            temp_file.close
-            temp_file.unlink
-          end
-        end
-
         protected
 
         def convert_image(source, target, format)
 
-          image_info = nil
-          image_info = MiniMagick::Image::Info.new(source) {|b| b.quiet} if @wm_image
+          if @page
+            image = MiniMagick::Image.open(source) { |b| b.quiet }
+            source = image.pages[@page].path if image.pages.size > 1
+          end
 
           MiniMagick::Tool::Convert.new do |convert|
-            convert.quiet if @quiet
-            if @wm_image
-              convert << @wm_image.path
-              convert.bordercolor('transparent').border("#{@wm_gap}%") if @wm_gap > 0
-              convert.filter('Lagrange')
-              convert.resize("#{image_info['width'] / @wm_tiles}x#{image_info['height'] / @wm_tiles}") if @wm_tiles > 0
-              convert.resize("#{@wm_resize}%") if @wm_resize
-              convert.write('mpr:watermark').delete.+
-            end
-
             convert.quiet if @quiet
             convert << source
             convert.flatten if @options[:flatten].nil? && format == :JPG
-            if @wm_image
-              if @wm_tiles >= 0 and @wm_tiles <= 1
-                convert << 'mpr:watermark'
-              else
-                convert.stack do |stack|
-                  stack.size("#{image_info['width']}x#{image_info['height']}")
-                  stack << 'xc:transparent'
-                  stack.tile('mpr:watermark')
-                  stack.draw "rectangle 0,0,#{image_info['width']},#{image_info['height']}"
-                end
-              end
-              convert.compose(@wm_composition).gravity(@wm_gravity).define("compose:args=#{@wm_opacity}%").composite
-            end
-
             @flags.each {|f, v| v.is_a?(TrueClass) ? convert.send(f).+ : convert.send(f)}
             if @delete_date
               convert << '+set' << 'modify-date' << '+set' << 'create-date'