libis-format 1.0.5 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9fd6394c2eb7964f14d60e8c0ae9b116b8b4e6c12e0a8ee8cffc6b1f94063d1
-  data.tar.gz: 60ef07f34501ce38ab9a3551009aeab0548cee6dbd02612ca2c6ec2da884d288
+  metadata.gz: 4b7038f7206f0eeef883d0571657efca3999970d64abb78c2507bdc3a92a0c95
+  data.tar.gz: 6225fc365aba08fb6307997ce0b2615c4ea1f4c20ae9cb3765ebecc8e7ea5211
 SHA512:
-  metadata.gz: 4d72580ea2c577507190fa7c8f05dc29edb01e76ff16a8f55669575ca52006eafd26ca9d184628ea9b69d292a9b15b98289722cb25e17ec5d71df496114b1116
-  data.tar.gz: 72ef4b7c04340d86f61fb4211cc4532147c3933a16fefabadce7eacd524981b012ba1dfcdf1e52a29f7ff287488b1bc73ff846de9d04eb3afaea243e91a5a104
+  metadata.gz: dcce329e2ad5a6e3991e3d7eca4a35d6dc19b96d9cd83ff109bb744d7c51238ed66a9484000b260b007778df42a00ff1fc4077d387752d72fec25bebe221ed31
+  data.tar.gz: 77328b76c2703a8611638bbfbf0b9bf19996f694a04389191254c896471481ffb4ed87e57de7a000dfc67d106a281277e3226f85cb93ee08d44df4b7c526f383

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /spec/work/
+/spec/tmp/
 /tmp/
 *.bundle
 *.so
@@ -15,4 +16,7 @@
 mkmf.log
 /.idea/
 *.pyc
-tools/pdf/
+tools/pdf/
+/test*
+/*~
+/*.pdf

data/.travis.yml

@@ -1,41 +1,45 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
+sudo: false
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
   - sudo apt-get update -y -q
-  - sudo apt-get install -y -q ffmpeg gstreamer0.10-ffmpeg
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
   - sudo pip install --upgrade pip
   - sudo pip install --upgrade --ignore-installed opf-fido six
   - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
@@ -43,8 +47,12 @@ before_install:
   - sudo unzip droid.zip -d /opt/droid
   - sudo chmod 755 /opt/droid/droid.sh
   - sudo locale-gen "en_US.UTF-8"
-  - sudo dpkg-reconfigure locales
-  - echo "8\n9\n" | sudo apt-get install tzdata
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
 env:
   - LC_ALL=en_US.UTF-8
   - LANG=en_US.UTF-8

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -1,13 +1,35 @@
 FROM ubuntu:18.04
-RUN apt-get -q update
-RUN apt-get -q -y install wget unzip imagemagick python2.7 python-pip openjdk-8-jre-headless libreoffice --no-install-recommends
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+        wget \
+        unzip \
+        imagemagick \
+        python2.7 \
+        python-pip \
+        openjdk-8-jre-headless \
+        libreoffice \
+        ghostscript
+
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
 RUN pip install --upgrade pip && pip install opf-fido
+
 RUN wget -q -O droid.zip "http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip" \
  && mkdir -p /opt/droid \
  && unzip -d /opt/droid droid.zip \
  && chmod 755 /opt/droid/droid.sh \
  && rm droid.zip
+
 RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgraphics/fop/binaries/fop-2.3-bin.tar.gz&action=download" \
  && mkdir -p /opt/fop \
  && tar xzf fop.tar.gz -C /opt/fop --strip-components=2 \
  && rm fop.tar.gz
+
+RUN echo "8\n9\n" | apt-get install tzdata
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
+CMD /bin/bash

data/base/Dockerfile.alpine

@@ -0,0 +1,20 @@
+FROM alpine:3.8
+LABEL maintainer="kris.dekeyser@libis.be"
+RUN apk update -q && apk upgrade -q
+RUN apk add -q python2 py-pip openjdk8-jre ffmpeg libreoffice ttf-liberation ttf-freefont ttf-opensans sox git wget unzip sudo bash curl tar imagemagick
+# install fido
+RUN pip install -q --upgrade pip && pip install -q opf-fido
+# install droid
+RUN wget -q -O droid.zip http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip \
+ && mkdir -p /opt/droid \
+ && unzip -q -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+# install ImageMagick
+RUN apk add -q imagemagick
+ADD ../tools /tools
+ADD ../data /cfg
+VOLUME /data
+WORKDIR /data
+
+CMD ["/bin/bash", "-l"]

data/base/Dockerfile.rvm

@@ -0,0 +1,56 @@
+ARG ubuntu_ver=latest
+ARG ruby_ver=2.5.3
+ARG node_ver=10.15.0
+
+FROM ubuntu:${ubuntu_ver}
+LABEL maintainer="kris.dekeyser@libis.be"
+
+ENV RUBY_ENV=${ruby_ver}
+ENV NVM_DIR=/usr/local/nvm
+ENV NODE_VER=${node_ver}
+ENV NODE_PATH=$NVM_DIR/v$NODE_VER/lib/node_modules
+ENV NODE_PATH=$NVM_DIR/versions/node/v$NODE_VER/bin
+ENV PATH "${PATH}:${NODE_PATH}:${NODE_PATH}/npm"
+
+RUN echo $RUBY_ENV
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -q \
+ && apt-get install -y -q --no-install-recommends software-properties-common apt-utils
+RUN apt-add-repository -y ppa:rael-gc/rvm \
+ && apt-get update -q
+RUN apt-get  install -y -q --no-install-recommends \
+        build-essential \
+        gnupg2 \
+        git \
+        wget \
+        curl \
+        sudo \
+        unzip \
+        openjdk-8-jre-headless
+
+ENV BASH_ENV='/etc/profile exec bash'
+SHELL ["/bin/bash", "-l", "-c"]
+
+RUN addgroup --gid 2000 teneo \
+ && adduser --uid 2000 --gid 2000 --disabled-password --gecos "Teneo user" teneo \
+ && usermod -L -s /bin/bash teneo
+RUN echo 'teneo ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+USER teneo
+
+RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+
+RUN curl -sSL https://get.rvm.io | bash -s stable
+
+RUN echo 'gem: --no-document' > ${HOME}/.gemrc
+RUN echo rvm install ${ruby_ver}
+RUN echo rvm use ${RUBY_ENV}@teneo --create --ruby-version --ruby-gems
+
+# required due to bug in rubygems (https://github.com/rubygems/rubygems/issues/2483)
+RUN gem install rdoc bundler
+RUN gem update --no-document --system
+RUN rvm info
+#ADD . /home/libis/libis_format
+#WORKDIR /home/libis/libis_format
+#RUN bundle install && bundle exec rake spec

data/base/rework_path

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CMD=$1
+shift
+
+readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
+
+args=("$@")
+#echo "args: ${args[@]}"
+for map in "${mapping[@]}"
+do
+        map="${map//[$'\t\r\n ']}"
+        readarray -d '=' -t m <<< "$map"
+        src="${m[0]//[$'\t\r\n ']}"
+        tgt="${m[1]//[$'\t\r\n ']}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
+done
+${CMD} "${args[@]}"

data/docker_cfg.yml

@@ -0,0 +1 @@
+:fop_jar: '/tools/fop.jar'

data/lib/libis/format.rb

@@ -3,8 +3,11 @@ require 'libis/format/version'
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
-    autoload :TypeDatabaseImpl, 'libis/format/type_database_impl'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'
@@ -15,4 +18,10 @@ module Libis
     TOOL_DIR = File.join(ROOT_DIR, 'tools')
 
   end
-end
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? !!empty? : !self
+  end unless defined? :blank?
+end

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -23,23 +24,26 @@ module Libis
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
         droid: 20 * 60,
-        ffmpeg: 5 * 60,
+        ffmpeg: 500 * 60,
         fido: 20 * 60,
-        file_tool: 5 * 60,
-        fop: 5 * 60,
-        identification_tool: 5 * 60,
-        office_to_pdf: 5 * 60,
-        pdf_copy: 5 * 60,
-        pdf_merge: 5 * 60,
-        pdf_optimizer: 5 * 60,
-        pdf_split: 5 * 60,
-        pdf_to_pdfa: 5 * 60,
-        pdfa_validator: 5 * 60,
+        file_tool: 20 * 60,
+        fop: 50 * 60,
+        identification_tool: 50 * 60,
+        office_to_pdf: 50 * 60,
+        pdf_select: 50 * 60,
+        pdf_copy: 50 * 60,
+        pdf_merge: 50 * 60,
+        pdf_optimizer: 50 * 60,
+        pdf_split: 50 * 60,
+        pdf_watermark: 50 * 60,
+        pdf_to_pdfa: 50 * 60,
+        pdfa_validator: 50 * 60,
     }
 
   end

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,47 +69,13 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
+          convert_file(source, target)
 
-          else
-
-            convert_file(source, target)
-
-          end
-
-          target
-
-        end
-
-        def assemble_and_convert(sources, target)
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            target = convert_file(f.to_path, target)
-          end
-          target
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,8 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/tools/extend/string'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +23,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +65,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +87,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +105,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end