libis-format 1.3.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8f7ecb168b9a5db9a47b91035c1f8988c8a079963ad0764e23ad00b2bde61b1
-  data.tar.gz: 2fecf5a7eb4f0a74573d8182111dac6bb4abc05e9c8d22c248b79cc6533a93ef
+  metadata.gz: f563ad694fd3d0c9e4a9f0da3309c99ac4b103e6027fcc9cda1606f0f67549c2
+  data.tar.gz: 5cee3f4b3bb1e51db492ee14bf919b0ce0fa69e267292380364c32b4eec40b4e
 SHA512:
-  metadata.gz: 4ea0eced4ca98790237cbb936cd347964693a0b5d627e9564fb184621d018868d69e6208c5afc2cbaf78db096e5a7342fa42af9604921bf411b3b2c70c4e8722
-  data.tar.gz: 339f5f89b61747862d2f005ede140300f09ae8ab9aee4693d0214edac3bfba52b8f8b2eaa04b9e1e499debdee805403f1e8434c3ef7907d4ea0ae86e54bdc01f
+  metadata.gz: 409ea1fac8f8be4d2519f189090207b6a550a1b1ce174a71fa1886c69b667001459d4bdccd850dd55373199ae5553ebd02183e15eebc88a73d0ac3f8093f7d12
+  data.tar.gz: d3e4d77ddcaab1f5050aaf4acb02e74a221566abedb94062254efbc24d18703c8fdd739644b1cd5435b54a4497e77c82bdcbcb218d121bfaf135e483662aca06

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-ci
+repo_token: DRqohi64MRd8offoQKjjqqPlWV1vnm1mY

data/.gitignore

@@ -0,0 +1,20 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/spec/work/
+/spec/tmp/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+/.idea/
+*.pyc
+tools/pdf/
+/test*

data/.travis.yml

@@ -0,0 +1,70 @@
+dist: bionic
+language: ruby
+cache: bundler
+sudo: false
+rvm:
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+  - ruby-head
+  - jruby-9.1
+  - jruby-9.2
+  - jruby-head
+jdk:
+  - openjdk10
+  - openjdk11
+matrix:
+  exclude:
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk10
+    - rvm: ruby-head
+      jdk: openjdk10
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head
+branches:
+  only:
+    - master
+    - v2
+before_install:
+  - gem update --system
+  - gem --version
+  - sudo add-apt-repository ppa:mc3man/bionic-media -y
+  - sudo apt-get update -y -q
+  - sudo apt-get install
+  - sudo apt-get install -y -q ffmpeg ubuntu-restricted-extras
+  - sudo pip install --upgrade pip
+  - sudo pip install --upgrade --ignore-installed opf-fido six
+  - sudo wget -O droid.zip -q http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip
+  - sudo mkdir -p /opt/droid
+  - sudo unzip droid.zip -d /opt/droid
+  - sudo chmod 755 /opt/droid/droid.sh
+  - sudo locale-gen "en_US.UTF-8"
+  - echo 'Europe/Brussels' | sudo tee /etc/timezone
+  - export TZ=Europe/Brussels
+  - sudo dpkg-reconfigure --frontend noninteractive tzdata
+  - date
+before_script:
+  - export TZ=Europe/Brussels
+env:
+  - LC_ALL=en_US.UTF-8
+  - LANG=en_US.UTF-8
+addons:
+  apt:
+    packages:
+      - python2.7
+      - imagemagick
+      - libreoffice
+      - ghostscript
+      - sox
+      - libsox-dev
+      - libsox-fmt-all
+      - libchromaprint-dev
+      - libchromaprint-tools

data/Gemfile

@@ -1,17 +1,5 @@
-# frozen_string_literal: true
-
 source 'https://rubygems.org'
 
 gem 'coveralls', group: :test, require: false
 
 gemspec name: 'libis-format', development_group: :test
-
-group :development do
-  gem 'byebug'
-  gem 'debug'
-  gem 'standard'
-end
-
-# Gem publishing and changelog
-gem 'gem-release'
-gem 'github_changelog_generator'

data/README.md

@@ -26,9 +26,9 @@ Or install it yourself as:
 
 ## Usage
 
-### ::Libis::Format::TypeDatabase
+### ::Libis::Format::Library
 
-The type database is the core of the format services. It stores information about all the known formats.
+The format database is the core of the format services. It stores information about all the known formats.
 
 ### ::Libis::Format::Identifier
 

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+
+

data/base/Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+        wget \
+        unzip \
+        imagemagick \
+        python2.7 \
+        python-pip \
+        openjdk-8-jre-headless \
+        libreoffice \
+        ghostscript
+
+RUN apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip && pip install opf-fido
+
+RUN wget -q -O droid.zip "http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip" \
+ && mkdir -p /opt/droid \
+ && unzip -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+
+RUN wget -q -O fop.tar.gz "http://www.apache.org/dyn/closer.cgi?filename=/xmlgraphics/fop/binaries/fop-2.3-bin.tar.gz&action=download" \
+ && mkdir -p /opt/fop \
+ && tar xzf fop.tar.gz -C /opt/fop --strip-components=2 \
+ && rm fop.tar.gz
+
+RUN echo "8\n9\n" | apt-get install tzdata
+RUN mkdir -p /teneo/format/spec
+ADD ./tools /teneo/format/tools
+ADD ./data /teneo/format/data
+ADD ./spec/data /teneo/format/spec/data
+ADD ./base/rework_path /teneo/format/rework_path
+CMD /bin/bash

data/base/Dockerfile.alpine

@@ -0,0 +1,20 @@
+FROM alpine:3.8
+LABEL maintainer="kris.dekeyser@libis.be"
+RUN apk update -q && apk upgrade -q
+RUN apk add -q python2 py-pip openjdk8-jre ffmpeg libreoffice ttf-liberation ttf-freefont ttf-opensans sox git wget unzip sudo bash curl tar imagemagick
+# install fido
+RUN pip install -q --upgrade pip && pip install -q opf-fido
+# install droid
+RUN wget -q -O droid.zip http://www.nationalarchives.gov.uk/documents/information-management/droid-binary-6.4-bin.zip \
+ && mkdir -p /opt/droid \
+ && unzip -q -d /opt/droid droid.zip \
+ && chmod 755 /opt/droid/droid.sh \
+ && rm droid.zip
+# install ImageMagick
+RUN apk add -q imagemagick
+ADD ../tools /tools
+ADD ../data /cfg
+VOLUME /data
+WORKDIR /data
+
+CMD ["/bin/bash", "-l"]

data/base/Dockerfile.rvm

@@ -0,0 +1,56 @@
+ARG ubuntu_ver=latest
+ARG ruby_ver=2.5.3
+ARG node_ver=10.15.0
+
+FROM ubuntu:${ubuntu_ver}
+LABEL maintainer="kris.dekeyser@libis.be"
+
+ENV RUBY_ENV=${ruby_ver}
+ENV NVM_DIR=/usr/local/nvm
+ENV NODE_VER=${node_ver}
+ENV NODE_PATH=$NVM_DIR/v$NODE_VER/lib/node_modules
+ENV NODE_PATH=$NVM_DIR/versions/node/v$NODE_VER/bin
+ENV PATH "${PATH}:${NODE_PATH}:${NODE_PATH}/npm"
+
+RUN echo $RUBY_ENV
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -q \
+ && apt-get install -y -q --no-install-recommends software-properties-common apt-utils
+RUN apt-add-repository -y ppa:rael-gc/rvm \
+ && apt-get update -q
+RUN apt-get  install -y -q --no-install-recommends \
+        build-essential \
+        gnupg2 \
+        git \
+        wget \
+        curl \
+        sudo \
+        unzip \
+        openjdk-8-jre-headless
+
+ENV BASH_ENV='/etc/profile exec bash'
+SHELL ["/bin/bash", "-l", "-c"]
+
+RUN addgroup --gid 2000 teneo \
+ && adduser --uid 2000 --gid 2000 --disabled-password --gecos "Teneo user" teneo \
+ && usermod -L -s /bin/bash teneo
+RUN echo 'teneo ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+USER teneo
+
+RUN gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+
+RUN curl -sSL https://get.rvm.io | bash -s stable
+
+RUN echo 'gem: --no-document' > ${HOME}/.gemrc
+RUN echo rvm install ${ruby_ver}
+RUN echo rvm use ${RUBY_ENV}@teneo --create --ruby-version --ruby-gems
+
+# required due to bug in rubygems (https://github.com/rubygems/rubygems/issues/2483)
+RUN gem install rdoc bundler
+RUN gem update --no-document --system
+RUN rvm info
+#ADD . /home/libis/libis_format
+#WORKDIR /home/libis/libis_format
+#RUN bundle install && bundle exec rake spec

data/base/rework_path

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CMD=$1
+shift
+
+readarray -d ',' -t mapping <<< "$VOLUME_MAPPING"
+
+args=("$@")
+#echo "args: ${args[@]}"
+for map in "${mapping[@]}"
+do
+        map="${map//[$'\t\r\n ']}"
+        readarray -d '=' -t m <<< "$map"
+        src="${m[0]//[$'\t\r\n ']}"
+        tgt="${m[1]//[$'\t\r\n ']}"
+#        echo "$src -> $tgt"
+        args=("${args[@]/$src/$tgt}")
+#        echo "args: ${args[@]}"
+done
+${CMD} "${args[@]}"

data/bin/{pdf_tool → pdf_copy}

@@ -5,10 +5,9 @@ require 'libis-tools'
 
 ::Libis::Tools::Config.logger.level = :WARN
 
-command = ARGV.shift
-command = command.split('_')
 source = ARGV.shift
 target = ARGV.shift
 options = ARGV
 
-::Libis::Format::Tool::PdfTool.run command, source, target, *options
+::Libis::Format::Tool::PdfCopy.run source, target, options
+

data/data/PDFA_def.ps

@@ -5,10 +5,10 @@
 % unless the user modifies the corresponding line below.
 
 % Define entries in the document Info dictionary :
-/ICCProfile ([**ICC profile**])
+/ICCProfile ([** Fill in ICC profile location **])
 def
 
-[ [**METADATA**]
+[ /Title (Title)
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -35,6 +35,6 @@ def
   /Type /OutputIntent             % Must be so (the standard requires).
   /S /GTS_PDFA1                   % Must be so (the standard requires).
   /DestOutputProfile {icc_PDFA}            % Must be so (see above).
-  /OutputConditionIdentifier ([**ICC reference**])
+  /OutputConditionIdentifier ([** Fill in ICC reference name **])
 >> /PUT pdfmark
 [{Catalog} <</OutputIntents [ {OutputIntent_PDFA} ]>> /PUT pdfmark

data/data/types.yml

@@ -96,14 +96,14 @@ AUDIO:
 
   AAC:
     NAME: Advanced Audio Coding (AAC)
-    MIME: audio/aac audio/x-hx-aac-adts
+    MIME: audio/aac
     EXTENSIONS: aac
 
   WMA:
     NAME: Windows Media Audio (WMA)
     MIME: audio/x-ms-wma
     PUID: fmt/132
-    EXTENSIONS: wma asf
+    EXTENSIONS: wma
 
   ALAC:
     NAME: Apple Lossless Audio Codec (ALAC)
@@ -194,7 +194,7 @@ VIDEO:
     NAME: Windows Media Video (WMV)
     MIME: video/x-ms-wmv
     PUID: fmt/133 fmt/441
-    EXTENSIONS: wmv asf
+    EXTENSIONS: wmv
 
   DV:
     NAME: Digital Video (DV)
@@ -319,12 +319,6 @@ TEXT:
 
 TABULAR:
 
-  CSV:
-    NAME: Comma Separated Values
-    MIME: text/csv
-    PUID: x-fmt/18
-    EXTENSIONS: csv
-
   MSXLS:
     NAME: Microsoft Excel Spreadsheet (XLS)
     MIME: application/vnd.ms-excel,application/msexcel
@@ -357,15 +351,8 @@ TABULAR:
 
 PRESENTATION:
 
-  MSPPS:
-    NAME: Microsoft Powerpoint Presentation Show
-    PUID: x-fmt/87
-    MIME: application/vnd.ms-powerpoint
-    EXTENSIONS: pps
-
   MSPPT:
     NAME: Microsoft Powerpoint Presentation (PPT)
-    PUID: fmt/181 fmt/179 fmt/1747 fmt/1748 x-fmt/88 fmt/125 fmt/126
     MIME: application/vnd.ms-powerpoint,application/mspowerpoint
     EXTENSIONS: ppt
 
@@ -377,7 +364,7 @@ PRESENTATION:
 
   OO_IMPRESS:
     NAME: OpenDocument Presentation (ODP)
-    PUID: fmt/138 fmt/292 fmt/293
+    PUIID: fmt/138 fmt/292 fmt/293
     MIME: application/vnd.oasis.opendocument.presentation application/vnd.oasis.opendocument.presentation-template
     EXTENSIONS: odp otp
 

data/docker_cfg.yml

@@ -0,0 +1 @@
+:fop_jar: '/tools/fop.jar'

data/lib/libis/format/cli/convert.rb

@@ -67,7 +67,7 @@ module Libis
         arguments that any of the targetted converters support.
 
         The source file's format will be identified by the Libis::Format::Identifier and the target file's format
-        will be derived from the file's extension. The Libis::Format::TypeDatabase is used to relate extensions
+        will be derived from the file's extension. The Libis::Format::Library is used to relate extensions
         with formats.
 
         DESC
@@ -89,7 +89,7 @@ module Libis
             A source file name and target file name should be supplied. The source file should exist and be readable.
             The target file should be writable, but should not exist.
 
-            The target file's format will be derived from the file's extension. The Libis::Format::TypeDatabase is used 
+            The target file's format will be derived from the file's extension. The Libis::Format::Library is used 
             to relate extensions with formats.
 
         DESC
@@ -265,11 +265,11 @@ module Libis
         end
 
         def extname(format)
-          Libis::Format::TypeDatabase.type_extentions(format).first
+          Libis::Format::Library.get_field(format, :extensions)
         end
 
         def get_format(file_name)
-          Libis::Format::TypeDatabase.ext_types(File.extname(file_name)).first
+          Libis::Format::Library.get_field_by(:extension, File.extname(file_name), :format)
         end
 
         def format_identifier(file)

data/lib/libis/format/cli/prompt_helper.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'tty-prompt'
 require 'pastel'
 
@@ -7,6 +5,7 @@ module Libis
   module Format
     module Cli
       module PromptHelper
+
         attr_reader :prompt, :pastel
 
         def initialize(*args)
@@ -15,6 +14,8 @@ module Libis
           super
         end
 
+        protected
+
         private
 
         def index_of(list, value)
@@ -24,9 +25,7 @@ module Libis
         end
 
         def ask(question, bool: false, enum: nil, default: nil, mask: false)
-          cmd = :ask
-          args = [question]
-          opts = {}
+          cmd, args, opts = :ask, [question], {}
           if enum
             cmd = :select
             args << enum
@@ -39,57 +38,50 @@ module Libis
           prompt.send(cmd, *args, opts)
         end
 
-        def tree_select(path, question: nil, file: false, page_size: 22, filter: true, cycle: false, create: false, # rubocop:disable Metrics/ParameterLists
+        def tree_select(path, question: nil, file: false, page_size: 22, filter: true, cycle: false, create: false,
                         default_choices: nil)
           path = Pathname.new(path) unless path.is_a? Pathname
 
           return path unless path.exist?
-
           path = path.realpath
 
           dirs = path.children.select(&:directory?).sort
           files = file ? path.children.select(&:file?).sort : []
 
           choices = []
-          choices << { name: "Folder: #{path}", value: path, disabled: file ? '' : false }
+          choices << {name: "Folder: #{path}", value: path, disabled: file ? '' : false}
           choices += default_choices if default_choices
-          if create
-            choices << {
-              name: '-- new directory --',
-              value: lambda {
-                new_name = prompt.ask('new directory name:', modify: :trim, required: true)
-                new_path = path + new_name
-                FileUtils.mkdir(new_path.to_path)
-                new_path
-              }
-            }
+          choices << {name: '-- new directory --', value: -> do
+            new_name = prompt.ask('new directory name:', modify: :trim, required: true)
+            new_path = path + new_name
+            FileUtils.mkdir(new_path.to_path)
+            new_path
           end
+          } if create
 
-          if file && create
-            choices << {
-              name: '-- new file --', value: lambda {
-                new_name = prompt.ask('new file name:', modify: :trim, required: true)
-                path + new_name
-              }
-            }
+          choices << {name: "-- new file --", value: -> do
+            new_name = prompt.ask('new file name:', modify: :trim, required: true)
+            path + new_name
           end
+          } if file && create
 
-          choices << { name: '[..]', value: path.parent }
+          choices << {name: '[..]', value: path.parent}
 
-          dirs.each { |d| choices << { name: "[#{d.basename}]", value: d } }
-          files.each { |f| choices << { name: f.basename.to_path, value: f } }
+          dirs.each {|d| choices << {name: "[#{d.basename}]", value: d}}
+          files.each {|f| choices << {name: f.basename.to_path, value: f}}
 
           question ||= "Select #{'file or ' if files}directory"
           selection = prompt.select question, choices,
-                                    per_page: page_size, filter:, cycle:, default: file ? 2 : 1
+                                    per_page: page_size, filter: filter, cycle: cycle, default: file ? 2 : 1
 
           return selection unless selection.is_a? Pathname
           return selection.to_path if selection == path || selection.file?
 
-          tree_select selection, question:, file:, page_size:, filter:,
-                                 cycle:, create:, default_choices:
+          tree_select selection, question: question, file: file, page_size: page_size, filter: filter,
+                      cycle: cycle, create: create, default_choices: default_choices
         end
+
       end
     end
   end
-end
+end

data/lib/libis/format/command_line.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'thor'
 require 'tty-prompt'
 require 'tty-config'
@@ -9,7 +7,9 @@ require 'libis/format/cli/format'
 
 module Libis
   module Format
+
     class CommandLine < Thor
+
       def self.exit_on_failure?
         true
       end
@@ -20,5 +20,6 @@ module Libis
       desc 'format', 'perform format identification on a given file or directory'
       subcommand 'format', Cli::Format
     end
+
   end
 end

data/lib/libis/format/config.rb

@@ -1,10 +1,11 @@
-# frozen_string_literal: true
-
+# encoding: utf-8
 require 'os'
 require 'libis-tools'
 
 module Libis
   module Format
+
+    # noinspection RubyConstantNamingConvention
     Config = ::Libis::Tools::Config
 
     Config[:converter_chain_max_level] = 8
@@ -15,30 +16,33 @@ module Libis
     Config[:ghostscript_cmd] = 'gs'
     Config[:droid_cmd] = '/opt/droid/droid.sh'
     Config[:droid_temp_path] = '/tmp'
-    Config[:fido_cmd] = 'fido'
+    Config[:fido_cmd] = '/usr/local/bin/fido'
     Config[:fop_jar] = File.join(Libis::Format::TOOL_DIR, 'fop', 'build', 'fop.jar')
     Config[:ffmpeg_cmd] = 'ffmpeg'
-    Config[:fido_formats] = [File.join(Libis::Format::DATA_DIR, 'lias_formats.xml')]
+    Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
-    Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-3.0.3.jar')
-    Config[:wkhtmltopdf] = 'wkhtmltopdf'
+    Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
+    # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
-      droid: 20 * 60,
-      ffmpeg: 5 * 60,
-      fido: 20 * 60,
-      file_tool: 5 * 60,
-      fop: 5 * 60,
-      identification_tool: 5 * 60,
-      office_to_pdf: 5 * 60,
-      email2pdf: 5 * 60,
-      pdf_tool: 5 * 60,
-      pdf_optimizer: 5 * 60,
-      pdf_to_pdfa: 5 * 60,
-      pdfa_validator: 5 * 60
+        droid: 20 * 60,
+        ffmpeg: 5 * 60,
+        fido: 20 * 60,
+        file_tool: 5 * 60,
+        fop: 5 * 60,
+        identification_tool: 5 * 60,
+        office_to_pdf: 5 * 60,
+        pdf_copy: 5 * 60,
+        pdf_merge: 5 * 60,
+        pdf_optimizer: 5 * 60,
+        pdf_split: 5 * 60,
+        pdf_to_pdfa: 5 * 60,
+        pdfa_validator: 5 * 60,
     }
+
   end
 end