libis-format 1.0.4 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a96a71d90952cf5d8be62d9b82ac6fc22c4f080a
-  data.tar.gz: 79bda271ae8b628e96c7a3d19b65884a3fe1eee0
+SHA256:
+  metadata.gz: 7f89add0e88c5d7a136475b303e5eec4da7af65ae6469ed24487e00c900871aa
+  data.tar.gz: '0902a88e237a8cf7a42f0ec0ea6aac4370590649aca894853a4f5de8b12dbb1e'
 SHA512:
-  metadata.gz: 303f4353c79d988110668ddc1c4bb5867d4e644b30d1a6f60e24da13a2f8f99703ab1b8f5afba73ed11aee0cfdd8bad6e759e857350a97d734d021fcfca31c32
-  data.tar.gz: 612a76df7a9907900aa292e68065f3c13d1362edffa01cbf2425edb8c8f41040d17871680ad7bab430a49790da96bba323ce2706ab13dfa39494ca8795e7d639
+  metadata.gz: 90d3b575f8b87aca635d7b526d26ce8a899626b83950e66ed7df382b4e983cf93e3bab4526d5fdbdd268c94e158c99f9296858816f4f630a29d7a7ae89ea73e0
+  data.tar.gz: dfb7854cacec55c33fe6e01d3cc834eb3debe1657366c0247803b3227e84113f9f230e6809462c212c659388b576faeed038464752f959997d39a84aabe52858

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /spec/work/
+/spec/tmp/
 /tmp/
 *.bundle
 *.so
@@ -15,4 +16,7 @@
 mkmf.log
 /.idea/
 *.pyc
-tools/pdf/
+tools/pdf/
+/test*
+/*~
+/*.pdf

data/.travis.yml

@@ -1,43 +1,61 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
+sudo: false
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
-  - sudo apt-get update -y
-  - sudo apt-get install -y ffmpeg gstreamer0.10-ffmpeg
-  - sudo pip install opf-fido --upgrade --ignore-installed six
-
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
+  - sudo apt-get update -y -q
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
+  - sudo pip install --upgrade pip
+  - sudo pip install --upgrade --ignore-installed opf-fido six
+  - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
+  - sudo mkdir -p /opt/droid
+  - sudo unzip droid.zip -d /opt/droid
+  - sudo chmod 755 /opt/droid/droid.sh
+  - sudo locale-gen "en_US.UTF-8"
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
+env:
+  - LC_ALL=en_US.UTF-8
+  - LANG=en_US.UTF-8
 addons:
   apt:
     packages:

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+        wget \
+        unzip \
+        imagemagick \
+        python2.7 \
+        python-pip \
+        openjdk-8-jre-headless \
+        libreoffice \
+        ghostscript
+
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip && pip install opf-fido
+
+RUN wget -q -O droid.zip "http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip" \
+ && mkdir -p /opt/droid \
+ && unzip -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+
+RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgraphics/fop/binaries/fop-2.3-bin.tar.gz&action=download" \
+ && mkdir -p /opt/fop \
+ && tar xzf fop.tar.gz -C /opt/fop --strip-components=2 \
+ && rm fop.tar.gz
+
+RUN echo "8\n9\n" | apt-get install tzdata
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
+CMD /bin/bash

data/base/Dockerfile.alpine

@@ -0,0 +1,20 @@
+FROM alpine:3.8
+LABEL maintainer="kris.dekeyser@libis.be"
+RUN apk update -q && apk upgrade -q
+RUN apk add -q python2 py-pip openjdk8-jre ffmpeg libreoffice ttf-liberation ttf-freefont ttf-opensans sox git wget unzip sudo bash curl tar imagemagick
+# install fido
+RUN pip install -q --upgrade pip && pip install -q opf-fido
+# install droid
+RUN wget -q -O droid.zip http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip \
+ && mkdir -p /opt/droid \
+ && unzip -q -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+# install ImageMagick
+RUN apk add -q imagemagick
+ADD ../tools /tools
+ADD ../data /cfg
+VOLUME /data
+WORKDIR /data
+
+CMD ["/bin/bash", "-l"]

data/base/Dockerfile.rvm

@@ -0,0 +1,56 @@
+ARG ubuntu_ver=latest
+ARG ruby_ver=2.5.3
+ARG node_ver=10.15.0
+
+FROM ubuntu:${ubuntu_ver}
+LABEL maintainer="kris.dekeyser@libis.be"
+
+ENV RUBY_ENV=${ruby_ver}
+ENV NVM_DIR=/usr/local/nvm
+ENV NODE_VER=${node_ver}
+ENV NODE_PATH=$NVM_DIR/v$NODE_VER/lib/node_modules
+ENV NODE_PATH=$NVM_DIR/versions/node/v$NODE_VER/bin
+ENV PATH "${PATH}:${NODE_PATH}:${NODE_PATH}/npm"
+
+RUN echo $RUBY_ENV
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -q \
+ && apt-get install -y -q --no-install-recommends software-properties-common apt-utils
+RUN apt-add-repository -y ppa:rael-gc/rvm \
+ && apt-get update -q
+RUN apt-get  install -y -q --no-install-recommends \
+        build-essential \
+        gnupg2 \
+        git \
+        wget \
+        curl \
+        sudo \
+        unzip \
+        openjdk-8-jre-headless
+
+ENV BASH_ENV='/etc/profile exec bash'
+SHELL ["/bin/bash", "-l", "-c"]
+
+RUN addgroup --gid 2000 teneo \
+ && adduser --uid 2000 --gid 2000 --disabled-password --gecos "Teneo user" teneo \
+ && usermod -L -s /bin/bash teneo
+RUN echo 'teneo ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+USER teneo
+
+RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+
+RUN curl -sSL https://get.rvm.io | bash -s stable
+
+RUN echo 'gem: --no-document' > ${HOME}/.gemrc
+RUN echo rvm install ${ruby_ver}
+RUN echo rvm use ${RUBY_ENV}@teneo --create --ruby-version --ruby-gems
+
+# required due to bug in rubygems (https://github.com/rubygems/rubygems/issues/2483)
+RUN gem install rdoc bundler
+RUN gem update --no-document --system
+RUN rvm info
+#ADD . /home/libis/libis_format
+#WORKDIR /home/libis/libis_format
+#RUN bundle install && bundle exec rake spec

data/base/rework_path

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CMD=$1
+shift
+
+readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
+
+args=("$@")
+#echo "args: ${args[@]}"
+for map in "${mapping[@]}"
+do
+        map="${map//[$'\t\r\n ']}"
+        readarray -d '=' -t m <<< "$map"
+        src="${m[0]//[$'\t\r\n ']}"
+        tgt="${m[1]//[$'\t\r\n ']}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
+done
+${CMD} "${args[@]}"

data/docker_cfg.yml

@@ -0,0 +1 @@
+:fop_jar: '/tools/fop.jar'

data/lib/libis/format.rb

@@ -3,8 +3,11 @@ require 'libis/format/version'
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
-    autoload :TypeDatabaseImpl, 'libis/format/type_database_impl'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'
@@ -15,4 +18,10 @@ module Libis
     TOOL_DIR = File.join(ROOT_DIR, 'tools')
 
   end
-end
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? !!empty? : !self
+  end unless defined? :blank?
+end

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -9,20 +10,23 @@ module Libis
 
     Config[:converter_chain_max_level] = 8
 
-    Config[:java_path] = 'java'
-    Config[:j2kdriver] = 'j2kdriver'
-    Config[:soffice_path] = 'soffice'
-    Config[:ghostscript_path] = 'gs'
-    Config[:droid_path] = '/opt/droid/droid.sh'
-    Config[:fido_path] = '/usr/local/bin/fido'
-    Config[:fop_jar] = '/opt/fop/current/fop/build/fop.jar'
-    Config[:ffmpeg_path] = 'ffmpeg'
+    Config[:java_cmd] = 'java'
+    Config[:j2k_cmd] = 'j2kdriver'
+    Config[:soffice_cmd] = 'soffice'
+    Config[:ghostscript_cmd] = 'gs'
+    Config[:droid_cmd] = '/opt/droid/droid.sh'
+    Config[:droid_temp_path] = '/tmp'
+    Config[:fido_cmd] = '/usr/local/bin/fido'
+    Config[:fop_jar] = File.join(Libis::Format::TOOL_DIR, 'fop', 'build', 'fop.jar')
+    Config[:ffmpeg_cmd] = 'ffmpeg'
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
+    Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
-    Config[:raw_audio_convert_cmd] = 'sox %s -e signed -b 16 -t wav %s rate %d channels %d'
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
         droid: 20 * 60,
@@ -32,6 +36,7 @@ module Libis
         fop: 5 * 60,
         identification_tool: 5 * 60,
         office_to_pdf: 5 * 60,
+        pdf_select: 5 * 60,
         pdf_copy: 5 * 60,
         pdf_merge: 5 * 60,
         pdf_optimizer: 5 * 60,

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,50 +69,16 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
+          convert_file(source, target)
 
-          else
-
-            convert_file(source, target)
-
-          end
-
-          target
-
-        end
-
-        def assemble_and_convert(sources, target)
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            target = convert_file(f.to_path, target)
-          end
-          target
         end
 
-        def sounds_like(file1, file2, threshold, rate, channels)
+        def self.sounds_like(file1, file2, threshold, rate, channels)
           rate ||= 96000
           channels ||= 2
           threshold ||= 0.85

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,8 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/tools/extend/string'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +23,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +65,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +87,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +105,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end