libis-format 1.0.2 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 21f527308cef3b9865c24c956f5a76cf67973400
-  data.tar.gz: 19c867aef5d28da24a3354025b11fc718b885a7f
+SHA256:
+  metadata.gz: bfe52fc3216a0da8528941b4232c252880006137cc92fe8d42512840f5b7ab7d
+  data.tar.gz: 349303abd8b4d8748960713933f532bae3ca83d45613a79321269497baa6ae2a
 SHA512:
-  metadata.gz: 3af22a3b301bc89a0839215a96a571d729478c61f3d414ee144e7f011d16443f4e604efd8f6afb0b40d2dd02dc77ca65878dfba7b29a6f4bc186beec8373b437
-  data.tar.gz: 81bebc7836625f251e9365100bfdb4a13d3ec4424a26a536521bb1776a11d96472645af1e360077ff64d5ef4e6f08671e0c87330d8fa32f5230b390509a016ce
+  metadata.gz: '0917e66b03e3bf008dd3a86761f9004d93d1fbc3c02c8c692a1f6f312f25c4edc0a1536b73e4a01043b4e03c85125437ae1307c49349bdee7c46d3f5b2503231'
+  data.tar.gz: 29bd68dce1a8f5f2c0216299b29268d3fdcbafaed4282b7076fac4810762fb377a403dbc08dc23672a3eda4bb17cae8c8882bb24d21980396d3c2943935d1437

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /spec/work/
+/spec/tmp/
 /tmp/
 *.bundle
 *.so
@@ -15,4 +16,7 @@
 mkmf.log
 /.idea/
 *.pyc
-tools/pdf/
+tools/pdf/
+/test*
+/*~
+/*.pdf

data/.travis.yml

@@ -1,43 +1,61 @@
+dist: bionic
 language: ruby
-sudo: false
-dist: trusty
 cache: bundler
+sudo: false
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
+  - jruby-9.2
   - jruby-head
 jdk:
-  - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
-      - master
+    - master
+    - v2
 before_install:
-  - sudo add-apt-repository ppa:mc3man/trusty-media -y
-  - sudo apt-get update -y
-  - sudo apt-get install -y ffmpeg gstreamer0.10-ffmpeg
-  - sudo pip install opf-fido --upgrade --ignore-installed six
-
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
+  - sudo apt-get update -y -q
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
+  - sudo pip install --upgrade pip
+  - sudo pip install --upgrade --ignore-installed opf-fido six
+  - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
+  - sudo mkdir -p /opt/droid
+  - sudo unzip droid.zip -d /opt/droid
+  - sudo chmod 755 /opt/droid/droid.sh
+  - sudo locale-gen "en_US.UTF-8"
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
+env:
+  - LC_ALL=en_US.UTF-8
+  - LANG=en_US.UTF-8
 addons:
   apt:
     packages:

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/base/Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+        wget \
+        unzip \
+        imagemagick \
+        python2.7 \
+        python-pip \
+        openjdk-8-jre-headless \
+        libreoffice \
+        ghostscript
+
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip && pip install opf-fido
+
+RUN wget -q -O droid.zip "http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip" \
+ && mkdir -p /opt/droid \
+ && unzip -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+
+RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgraphics/fop/binaries/fop-2.3-bin.tar.gz&action=download" \
+ && mkdir -p /opt/fop \
+ && tar xzf fop.tar.gz -C /opt/fop --strip-components=2 \
+ && rm fop.tar.gz
+
+RUN echo "8\n9\n" | apt-get install tzdata
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
+CMD /bin/bash

data/base/Dockerfile.alpine

@@ -0,0 +1,20 @@
+FROM alpine:3.8
+LABEL maintainer="kris.dekeyser@libis.be"
+RUN apk update -q && apk upgrade -q
+RUN apk add -q python2 py-pip openjdk8-jre ffmpeg libreoffice ttf-liberation ttf-freefont ttf-opensans sox git wget unzip sudo bash curl tar imagemagick
+# install fido
+RUN pip install -q --upgrade pip && pip install -q opf-fido
+# install droid
+RUN wget -q -O droid.zip http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip \
+ && mkdir -p /opt/droid \
+ && unzip -q -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+# install ImageMagick
+RUN apk add -q imagemagick
+ADD ../tools /tools
+ADD ../data /cfg
+VOLUME /data
+WORKDIR /data
+
+CMD ["/bin/bash", "-l"]

data/base/Dockerfile.rvm

@@ -0,0 +1,56 @@
+ARG ubuntu_ver=latest
+ARG ruby_ver=2.5.3
+ARG node_ver=10.15.0
+
+FROM ubuntu:${ubuntu_ver}
+LABEL maintainer="kris.dekeyser@libis.be"
+
+ENV RUBY_ENV=${ruby_ver}
+ENV NVM_DIR=/usr/local/nvm
+ENV NODE_VER=${node_ver}
+ENV NODE_PATH=$NVM_DIR/v$NODE_VER/lib/node_modules
+ENV NODE_PATH=$NVM_DIR/versions/node/v$NODE_VER/bin
+ENV PATH "${PATH}:${NODE_PATH}:${NODE_PATH}/npm"
+
+RUN echo $RUBY_ENV
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -q \
+ && apt-get install -y -q --no-install-recommends software-properties-common apt-utils
+RUN apt-add-repository -y ppa:rael-gc/rvm \
+ && apt-get update -q
+RUN apt-get  install -y -q --no-install-recommends \
+        build-essential \
+        gnupg2 \
+        git \
+        wget \
+        curl \
+        sudo \
+        unzip \
+        openjdk-8-jre-headless
+
+ENV BASH_ENV='/etc/profile exec bash'
+SHELL ["/bin/bash", "-l", "-c"]
+
+RUN addgroup --gid 2000 teneo \
+ && adduser --uid 2000 --gid 2000 --disabled-password --gecos "Teneo user" teneo \
+ && usermod -L -s /bin/bash teneo
+RUN echo 'teneo ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+USER teneo
+
+RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+
+RUN curl -sSL https://get.rvm.io | bash -s stable
+
+RUN echo 'gem: --no-document' > ${HOME}/.gemrc
+RUN echo rvm install ${ruby_ver}
+RUN echo rvm use ${RUBY_ENV}@teneo --create --ruby-version --ruby-gems
+
+# required due to bug in rubygems (https://github.com/rubygems/rubygems/issues/2483)
+RUN gem install rdoc bundler
+RUN gem update --no-document --system
+RUN rvm info
+#ADD . /home/libis/libis_format
+#WORKDIR /home/libis/libis_format
+#RUN bundle install && bundle exec rake spec

data/base/rework_path

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CMD=$1
+shift
+
+readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
+
+args=("$@")
+#echo "args: ${args[@]}"
+for map in "${mapping[@]}"
+do
+        map="${map//[$'\t\r\n ']}"
+        readarray -d '=' -t m <<< "$map"
+        src="${m[0]//[$'\t\r\n ']}"
+        tgt="${m[1]//[$'\t\r\n ']}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
+done
+${CMD} "${args[@]}"

data/bin/libis_format

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'libis/format/command_line'
+
+Libis::Format::CommandLine.start(ARGV)

data/docker_cfg.yml

@@ -0,0 +1 @@
+:fop_jar: '/tools/fop.jar'

data/lib/libis/format.rb

@@ -3,7 +3,11 @@ require 'libis/format/version'
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'
@@ -14,4 +18,10 @@ module Libis
     TOOL_DIR = File.join(ROOT_DIR, 'tools')
 
   end
-end
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? !!empty? : !self
+  end unless defined? :blank?
+end

data/lib/libis/format/cli/convert.rb

@@ -0,0 +1,310 @@
+require 'awesome_print'
+require 'libis-format'
+require 'libis-tools'
+require 'libis/tools/extend/hash'
+require 'fileutils'
+require 'tmpdir'
+require 'yaml'
+
+require 'libis/format/converter/image_converter'
+
+require_relative 'sub_command'
+
+module Libis
+  module Format
+    module Cli
+      class Convert < SubCommand
+
+        no_commands do
+          def self.description(field)
+            "#{STRING_CONFIG[field]}." + (DEFAULT_CONFIG[field].nil? ? '' : " default: #{DEFAULT_CONFIG[field]}")
+          end
+        end
+
+        DEFAULT_CONFIG = {
+            scale: '100%',
+            resize: '100%',
+            wm_tiles: 4,
+            wm_resize: 1.0,
+            wm_gap: 0.2,
+            wm_rotation: 30,
+            wm_opacity: 0.1,
+            wm_gravity: 'center',
+            wm_composition: 'modulate'
+        }
+
+        STRING_CONFIG = {
+            page: 'Page number to select for multipage documents',
+            quiet: 'suppress all warning messages. Error messages are still reported',
+            scale: 'minify / magnify the image with pixel block averaging and pixel replication, respectively',
+            resize: 'Resize the image',
+            resample: 'Resize the image so that its rendered size remains the same as the original at the specified target resolution',
+            flatten: 'Create a canvas the size of the first images virtual canvas using the current -background color, and -compose each image in turn onto that canvas. Images falling outside that canvas is clipped',
+            delete_date: 'Remove modified date and created date metadata from the target image',
+            colorspace: 'Set the image colorspace',
+            profile: 'Specify color profile used for the image',
+            wm_text: 'Create watermark with given text',
+            wm_file: 'Create watermark from given file',
+            wm_tiles: 'Number of tiles of the watermark to distribute over the image. 0: no tiling, 1: watermark image scaled to fit image, n>1: at least n tiles horizontally/vertically, n<0: tile without recaling',
+            wm_resize: 'Resize the watermark image (fraction 0.0 - 1.0)',
+            wm_gap: 'Leave n % of whitespace between the watermark images. Similar effect can be achieved with wm_resize',
+            wm_rotation: 'Rotate the watermark text n degrees counterclockwise (0-360)',
+            wm_opacity: 'Opacity of the watermark (fraction 0.0 - 1.0)',
+            wm_gravity: 'Center point of the watermark overlay',
+            wm_composition: 'Set the type of image composition'
+        }
+
+        desc 'auto SOURCE TARGET [options]', 'Convert SOURCE file to TARGET file using auto algorithm'
+        long_desc <<-DESC
+  
+        'auto SOURCE TARGET [options]' will convert a file according to the best conversion chain algorithm.
+
+        A source file name and target file name should be supplied. The source file should exist and be readable.
+        The target file should be writable, but should not exist.
+
+        Optionally an options file name can be added to the command line. The options file should be a valid YAML
+        file that contains either a Hash or an Array of Hashes. The content should contain the required methods and 
+        arguments that any of the targetted converters support.
+
+        The source file's format will be identified by the Libis::Format::Identifier and the target file's format
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
+        with formats.
+
+        DESC
+
+        method_option :options, aliases: '-o', desc: 'Options file'
+
+        def auto(source_file, target_file)
+          options_file = options[:options]
+          opts = check_input(source_file, target_file, options_file)
+          output, converter = do_convert(source_file, target_file, opts)
+          prompt.ok "Output file '#{output}' created with converter #{converter}."
+        end
+
+        desc 'image SOURCE TARGET [options]', 'Convert SOURCE image to TARGET image using ImageConverter'
+        long_desc <<-DESC
+
+            'image SOURCE TARGET [options]' will convert a SOURCE image to TARGET image using the ImageConverter.
+
+            A source file name and target file name should be supplied. The source file should exist and be readable.
+            The target file should be writable, but should not exist.
+
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
+            to relate extensions with formats.
+
+        DESC
+
+        method_option :page, type: :numeric, aliases: '-p', desc: description(:page)
+        method_option :quiet, type: :boolean, aliases: '-q', desc: description(:quiet)
+        method_option :scale, type: :numeric, aliases: '-c', desc: description(:scale)
+        method_option :resize, type: :numeric, aliases: '-r', desc: description(:resize)
+        method_option :resample, type: :numeric, aliases: '-m', desc: description(:resample)
+        method_option :flatten, type: :boolean, aliases: '-f', desc: description(:flatten)
+        method_option :delete_date, type: :boolean, aliases: '-dd', desc: description(:delete_date)
+        method_option :colorspace, aliases: '-cs', desc: description(:colorspace)
+        method_option :profile, aliases: '-prof', desc: description(:profile)
+        method_option :wm_text, desc: description(:wm_text)
+        method_option :wm_file, desc: description(:wm_file)
+        method_option :wm_tiles, type: :numeric, desc: description(:wm_tiles)
+        method_option :wm_resize, type: :numeric, desc: description(:wm_resize)
+        method_option :wm_gap, type: :numeric, desc: description(:wm_gap)
+        method_option :wm_rotation, type: :numeric, desc: description(:wm_rotation)
+        method_option :wm_opacity, type: :numeric, desc: description(:wm_opacity)
+        method_option :wm_gravity, desc: description(:wm_gravity)
+        method_option :wm_composition, desc: description(:wm_composition)
+
+        def image(source_file, target_file)
+          check_input(source_file, target_file)
+          convert_image(source_file, target_file)
+          prompt.ok "Output file '#{target_file}' created with image converter."
+        end
+
+        protected
+
+        def do_convert(source_file, target_file, opts)
+          format_info = format_identifier(source_file)
+          src_mime = format_info[:mimetype]
+          source_format = format_info[:TYPE]
+          unless source_format
+            prompt.error "File item %s format (#{src_mime}) is not supported."
+            exit
+          end
+          target_format = get_format(target_file)
+          converterlist = []
+          temp_files = []
+          opts.each do |o|
+            o = o.dup
+            tgt_format = o.delete(:target_format) || target_format
+            tgt_file = tempname(source_file, tgt_format)
+            temp_files << tgt_file
+            begin
+              source_file, converter = convert_file(source_file, tgt_file, source_format, tgt_format, o)
+            rescue Exception => e
+              prompt.error "File conversion of '%s' from '%s' to '%s' failed: %s @ %s" %
+                               [source_file, source_format, tgt_format, e.message, e.backtrace.first]
+              exit
+            end
+            source_format = tgt_format
+            converterlist << converter
+          end
+          converter = converterlist.join(' + ')
+          if target_file
+            FileUtils.mkpath(File.dirname(target_file))
+            FileUtils.copy(source_file, target_file)
+          else
+            target_file = temp_files.pop
+          end
+          temp_files.each {|tmp_file| FileUtils.rm_f tmp_file}
+          [target_file, converter]
+        end
+
+        def convert_file(src_file, tgt_file, src_format, tgt_format, opts)
+          converter = Libis::Format::Converter::Repository.get_converter_chain(src_format, tgt_format, opts)
+
+          unless converter
+            prompt.error "Could not find converter for #{src_format} -> #{tgt_format} with #{opts}"
+            exit
+          end
+
+          converter_name = converter.to_s
+          prompt.say 'Converting file %s to %s with %s ' % [src_file, tgt_file, converter_name]
+          converted = converter.convert(src_file, tgt_file)
+
+          unless converted && converted == tgt_file
+            prompt.error 'File conversion failed (%s).', converter_name
+            return [nil, converter_name]
+          end
+
+          [tgt_file, converter_name]
+        end
+
+        def convert_image(source_file, target_file)
+          target_format = get_format(target_file)
+          converter = Libis::Format::Converter::ImageConverter.new
+          converter.page options[:page] if options[:page]
+          converter.quiet options[:quiet] if options[:quiet]
+          converter.scale options[:scale] if options[:scale]
+          converter.resize options[:resize] if options[:resize]
+          converter.resample options[:resample] if options[:resample]
+          converter.flatten if options[:flatten]
+          converter.delete_date if options[:delete_date]
+          converter.colorspace options[:colorspace] if options[:colorspace]
+          converter.profile options[:profile] if options[:profile]
+          if options[:wm_text] || options[:wm_file]
+            wm_options = {}
+            wm_options[:text] = options[:wm_text] if options[:wm_text]
+            wm_options[:file] = options[:wm_file] if options[:wm_file]
+            wm_options[:tiles] = options[:wm_tiles] if options[:wm_tiles]
+            wm_options[:resize] = options[:wm_resize] if options[:wm_resize]
+            wm_options[:gap] = options[:wm_gap] if options[:wm_gap]
+            wm_options[:rotation] = options[:wm_rotation] if options[:wm_rotation]
+            wm_options[:opacity] = options[:wm_opacity] if options[:wm_opacity]
+            wm_options[:gravity] = options[:wm_gravity] if options[:wm_gravity]
+            wm_options[:composition] = options[:wm_composition] if options[:wm_composition]
+            converter.watermark(wm_options)
+          end
+          converter.convert source_file, target_file, target_format
+        end
+
+        private
+
+        def check_input(source_file, target_file, options_file = nil)
+          # Check if source file exists and can be read
+          unless source_file && File.exist?(source_file) && File.readable?(source_file)
+            prompt.error "Fatal error trying to access source file '#{source_file}'."
+            exit
+          end
+          # Check if target file argument is supplied
+          unless target_file
+            prompt.error "Fatal error: no target file name supplied."
+            exit
+          end
+          # Check if target file directory can be created
+          begin
+            FileUtils.mkdir_p(File.dirname(target_file))
+          rescue SystemCallError => e
+            prompt.error "Fatal error trying to create folder for target file: #{e.message}"
+            exit
+          end
+
+          return [{}] unless options_file
+
+          # Check if options file exists and can be read
+          unless File.exist?(options_file) && File.readable?(options_file)
+            prompt.error "Fatal error trying to access options file '#{options_file}'."
+            exit
+          end
+
+          begin
+            opts = ::YAML.load_file(options_file)
+            opts = case opts
+                   when Hash
+                     [opts]
+                   when Array
+                     opts
+                   when NilClass
+                     [{}]
+                   else
+                     prompt.error "Options file contents should be a Hash or an Array of Hashes."
+                     exit
+                   end
+            opts.each {|h| h.key_strings_to_symbols!(recursive: true)}
+            opts
+          rescue Exception => e
+            prompt.error "Fatal error trying to parse options file: #{e.message}"
+            exit
+          end
+
+        end
+
+        def tempname(source_file, target_format)
+          # noinspection RubyResolve
+          Dir::Tmpname.create(
+              [File.basename(source_file, '.*'), ".#{extname(target_format)}"]
+          ) {}
+        end
+
+        def extname(format)
+          Libis::Format::Library.get_field(format, :extensions)
+        end
+
+        def get_format(file_name)
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
+        end
+
+        def format_identifier(file)
+          prompt.say "Identifying format of file '#{file}'"
+          result = Libis::Format::Identifier.get(file) || {}
+          process_messages(result)
+          format = result[:formats][file]
+          unless format[:mimetype]
+            prompt.warn "Could not determine MIME type. Using default 'application/octet-stream'."
+            result[:puid] ||= 'fmt/unknown'
+          end
+          prompt.say "#{file} format: #{format[:GROUP]}, #{format[:TYPE]} (puid: #{format[:puid]}) mimetype: #{format[:mimetype]}"
+          format
+        end
+
+        def process_messages(format_result)
+          format_result[:messages].each do |msg|
+            case msg[0]
+            when :debug
+              prompt.say msg[1]
+            when :info
+              prompt.say msg[1]
+            when :warn
+              prompt.warn msg[1]
+            when :error
+              prompt.error msg[1]
+            when :fatal
+              prompt.error msg[1]
+            else
+              prompt.say "#{msg[0]}: #{msg[1]}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end