plaintext 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a474b4e8e63d06f2d83a97a7ff09c3b3375b27ed
+  data.tar.gz: 2cc7395ee7ab12c5588b9c187823ba7e126d91bd
+SHA512:
+  metadata.gz: 7afd62607f1f46c95278952591f6bda1325ea153b1006dc0092cd0128b615ffa4a11668315084c327541fca5591e5195d30c2635e51ca7a4391c67bd89df8b8d
+  data.tar.gz: a64aa765178280422de0a05d201ee11b2779649184ff2d72572e223c2e1bf160ce1c0f12b256e59c8ae2b102bcba699e7fd051205326c80d8dfd51b6690aae65

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.ruby-version
+.idea

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install:
+  - sudo apt-get -qq update
+  - sudo apt-get install -y catdoc unrtf poppler-utils tesseract-ocr
+  - gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in plaintext.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

data/README.md

@@ -0,0 +1,127 @@
+# plaintext
+
+This gem wraps command line tools to extract plain text from typical files such as
+
+- PDF
+- RTF
+- MS Office
+    - Word (doc, docx)
+    - Excel (xsl, xslx)
+    - PowerPoint (ppt, pptx)
+- OpenOffice + Libre
+    - Presentation
+    - Text
+    - Spreadsheet
+- Image files (png, jpeg, tiff), such as screenshots and scanned documents, through character recognition (OCR)
+- Plaintext (txt)
+- Comma-separated values (csv)
+
+## Acknowledgements
+
+This gem bases on work by Jens Krämer / Planio, who originally provided it as a
+[patch for Redmine](https://www.redmine.org/issues/306). Now, it is a collaborative effort of
+both project management software providers [Planio](https://plan.io) and [OpenProject](https://openproject.org)
+as both systems tackle the identical challenge to extract plain text from attachment files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'plaintext'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install plaintext
+
+#### Rails
+
+In a Rails application save `plaintext.yml.example` in `config/plaintext.yml` and overwrite the settings to 
+your needs.
+
+Then load that configuration file in an initializer. Add the following lines to `config/initializers/plaintext.rb`:
+
+```ruby
+file_name = File.join([Rails.root.to_s, 'config', 'plaintext'])
+if File.file?(file_name)
+  config_file = File.read(file_name)
+  Plaintext::Configuration.load(config_file)
+end
+````
+
+#### Plain Ruby
+
+Please overwrite `Plaintext::Configuration.load`.
+
+### Linux
+
+On linux the default configuration should work. However, make sure that the following packages are installed
+
+    $ apt-get install catdoc unrtf poppler-utils tesseract-ocr
+
+### Mac OS X
+
+On Mac things are still not complete. Please help us to have the same capabilities as under Linux. Right now we cannot
+extract text from presentation and spreadsheets.
+
+Please use homebrew to install the missing command line tools.
+
+    $ brew install unrtf poppler tesseract
+    
+The `plaintext.yml` should look like this:
+    
+```yml
+pdftotext:
+  - /usr/local/bin/pdftotext
+  - -enc
+  - UTF-8
+  - __FILE__
+  - '-'
+
+unrtf:
+  - /usr/local/bin/unrtf
+  - --text
+  - __FILE__
+
+tesseract:
+  - /usr/local/bin/tesseract
+  - __FILE__
+  - stdout
+
+catdoc:
+  - /usr/bin/textutil
+  - -convert
+  - txt
+  - -stdout
+  - __FILE__
+```
+
+## Usage
+
+```ruby
+# `file` is of type File.
+# `content_type` is a String.
+fulltext = Plaintext::Resolver.new(file, content_type).text
+```
+
+## License
+
+The `plaintext` gem is free software; you can redistribute it and/or modify it under the terms of the GNU General 
+Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any 
+later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied 
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with the plugin. If not, see
+[www.gnu.org/licenses](https://www.gnu.org/licenses/).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/planio-gmbh/plaintext.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "plaintext"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/plaintext/codeset_util.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Plaintext
+  module CodesetUtil
+    def self.to_utf8(str, encoding)
+      return str if str.nil?
+      str.force_encoding('ASCII-8BIT')
+      if str.empty?
+        str.force_encoding('UTF-8')
+        return str
+      end
+      enc = (encoding.nil? || encoding.size == 0) ? 'UTF-8' : encoding
+      if enc.upcase != 'UTF-8'
+        str.force_encoding(enc)
+        str = str.encode('UTF-8', invalid: :replace,
+                         undef: :replace, replace: '?')
+      else
+        str.force_encoding('UTF-8')
+        if !str.valid_encoding?
+          str = str.encode('US-ASCII', invalid: :replace,
+                           undef: :replace, replace: '?').encode('UTF-8')
+        end
+      end
+      str
+    end
+  end
+end

data/lib/plaintext/configuration.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Plaintext
+  module Configuration
+    class << self
+      attr_accessor :config
+
+      # Returns a configuration setting
+      def [](name)
+        load if self.config.nil?
+        self.config[name]
+      end
+
+      def load(config_file = nil)
+        self.config = {}
+        return unless config_file
+
+        file_config = YAML::load(ERB.new(config_file).result)
+        if file_config.is_a?(Hash)
+          self.config = file_config
+        else
+          warn "`config_file` is not a valid Plaintext configuration file, ignoring."
+        end
+      end
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/doc_handler.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class DocHandler < ExternalCommandHandler
+    CONTENT_TYPES = [
+        'application/vnd.ms-word',
+        'application/msword'
+    ]
+    DEFAULT = [
+        '/usr/bin/catdoc', '-dutf-8', '__FILE__'
+    ]
+    def initialize
+      @content_types = CONTENT_TYPES
+      @command = Plaintext::Configuration['catdoc'] || DEFAULT
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/image_handler.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class ImageHandler < ExternalCommandHandler
+    CONTENT_TYPES = [
+        'image/jpeg',
+        'image/png',
+        'image/tiff'
+    ]
+    DEFAULT = [
+        '/usr/bin/tesseract', '__FILE__', 'stdout'
+    ].freeze
+    def initialize
+      @content_types = CONTENT_TYPES
+      @command = Plaintext::Configuration['tesseract'] || DEFAULT
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class PdfHandler < ExternalCommandHandler
+    DEFAULT = [
+        '/usr/bin/pdftotext', '-enc', 'UTF-8', '__FILE__', '-'
+    ].freeze
+    def initialize
+      @content_type = 'application/pdf'
+      @command = Plaintext::Configuration['pdftotext'] || DEFAULT
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/ppt_handler.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class PptHandler < ExternalCommandHandler
+    CONTENT_TYPES = [
+        'application/vnd.ms-powerpoint',
+        'application/powerpoint',
+    ]
+    DEFAULT = [
+        '/usr/bin/catppt', '-dutf-8', '__FILE__'
+    ]
+    def initialize
+      @content_types = CONTENT_TYPES
+      @command = Plaintext::Configuration['catppt'] || DEFAULT
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/rtf_handler.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class RtfHandler < ExternalCommandHandler
+    DEFAULT = [
+        '/usr/bin/unrtf', '--text', '__FILE__'
+    ].freeze
+    def initialize
+      @content_type = 'application/rtf'
+      @command = Plaintext::Configuration['unrtf'] || DEFAULT
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler/xls_handler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class XlsHandler < ExternalCommandHandler
+    CONTENT_TYPES = [
+      'application/vnd.ms-excel',
+      'application/excel'
+    ]
+    DEFAULT = [
+        '/usr/bin/xls2csv', '-dutf-8', '__FILE__'
+    ]
+    def initialize
+      @content_types = CONTENT_TYPES
+      @command = Plaintext::Configuration['xls2csv'] || DEFAULT
+    end
+    def text(*_)
+      if str = super
+        str.delete('"').gsub /,+/, ' '
+      end
+    end
+  end
+end

data/lib/plaintext/file_handler/external_command_handler.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Plaintext
+  class ExternalCommandHandler < FileHandler
+    # TODO: Extract this to a proper module
+    # Executes the given command through IO.popen and yields an IO object
+    # representing STDIN / STDOUT
+    #
+    # Due to how popen works the command will be executed directly without
+    # involving the shell if cmd is an array.
+    require 'fileutils'
+    def shellout(cmd, options = {}, &block)
+      mode = "r+"
+      IO.popen(cmd, mode) do |io|
+        io.set_encoding("ASCII-8BIT") if io.respond_to?(:set_encoding)
+        io.close_write unless options[:write_stdin]
+        block.call(io) if block_given?
+      end
+    end
+
+    FILE_PLACEHOLDER = '__FILE__'.freeze
+
+    def text(file)
+      cmd = @command.dup
+      cmd[cmd.index(FILE_PLACEHOLDER)] = Pathname(file).to_s
+      shellout(cmd){ |io| io.read }.to_s
+    end
+
+    def accept?(content_type)
+      super and available?
+    end
+
+    def available?
+      @command.present? and File.executable?(@command[0])
+    end
+
+    def self.available?
+      new.available?
+    end
+  end
+end

data/lib/plaintext/file_handler/plaintext_handler.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class PlaintextHandler < FileHandler
+    CONTENT_TYPES = %w(text/csv text/plain)
+    def initialize
+      @content_types = CONTENT_TYPES
+    end
+
+    def text(file)
+      Plaintext::CodesetUtil.to_utf8 IO.read(file), 'UTF-8'
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/docx_handler.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class DocxHandler < OfficeDocumentHandler
+    def initialize
+      super
+      @content_type = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+      @file_name = 'word/document.xml'
+      @namespace_uri = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/pptx_handler.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class PptxHandler < OfficeDocumentHandler
+    CONTENT_TYPES = [
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
+        'application/vnd.ms-powerpoint.template.macroEnabled.12'
+    ]
+
+    def initialize
+      super
+      @content_types = CONTENT_TYPES
+      @namespace_uri = 'http://schemas.openxmlformats.org/drawingml/2006/main'
+    end
+
+    def text(file)
+      slides = []
+      Zip::File.open(file) do |zip_file|
+        zip_file.each do |entry|
+          if entry.name =~ /slide(\d+)\.xml/
+            slides << [$1, xml_to_text(entry.get_input_stream)]
+          end
+        end
+      end
+      slides.sort!{|a, b| a.first <=> b.first}
+      slides.map(&:last).join ' '
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/xlsx_handler.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class XlsxHandler < OfficeDocumentHandler
+    def initialize
+      super
+      @content_type = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      @file_name = 'xl/sharedStrings.xml'
+      @namespace_uri = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/office_document_handler.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Plaintext
+  # Base class for extractors for MS Office formats
+  class OfficeDocumentHandler < ZippedXmlHandler
+    def initialize
+      super
+      @element = 't'
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler/opendocument_handler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Plaintext
+  # Extractor for Open / Libre Office formats
+  class OpendocumentHandler < ZippedXmlHandler
+    CONTENT_TYPES = [
+        'application/vnd.oasis.opendocument.presentation',
+        'application/vnd.oasis.opendocument.presentation-template',
+        'application/vnd.oasis.opendocument.text',
+        'application/vnd.oasis.opendocument.text-template',
+        'application/vnd.oasis.opendocument.spreadsheet',
+        'application/vnd.oasis.opendocument.spreadsheet-template'
+    ]
+    def initialize
+      super
+      @file_name = 'content.xml'
+      @content_types = CONTENT_TYPES
+      @element = 'p'
+      @namespace_uri = 'urn:oasis:names:tc:opendocument:xmlns:text:1.0'
+    end
+  end
+end

data/lib/plaintext/file_handler/zipped_xml_handler.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Plaintext
+  # Handler base class for XML based (MS / Open / Libre) office documents.
+  class ZippedXmlHandler < FileHandler
+    require 'zip'
+    require 'nokogiri'
+
+    class SaxDocument < Nokogiri::XML::SAX::Document
+      attr_reader :text
+
+      def initialize(text_element, text_namespace)
+        @element = text_element
+        @namespace_uri = text_namespace
+        @text = ''.dup
+        @is_text = false
+      end
+
+      # Handle each element, expecting the name and any attributes
+      def start_element_namespace(name, attrs = [], prefix = nil, uri = nil, ns = [])
+        if name == @element and uri == @namespace_uri
+          @is_text = true
+        end
+      end
+
+      # Any characters between the start and end element expected as a string
+      def characters(string)
+        @text << string if @is_text
+      end
+
+      # Given the name of an element once its closing tag is reached
+      def end_element_namespace(name, prefix = nil, uri = nil)
+        if name == @element and uri == @namespace_uri
+          @text << ' '
+          @is_text = false
+        end
+      end
+    end
+
+    def text(file)
+      Zip::File.open(file) do |zip_file|
+        zip_file.each do |entry|
+          if entry.name == @file_name
+            return xml_to_text entry.get_input_stream
+          end
+        end
+      end
+    end
+
+    private
+
+    def xml_to_text(io)
+      sax_doc = SaxDocument.new @element, @namespace_uri
+      Nokogiri::XML::SAX::Parser.new(sax_doc).parse(io)
+      sax_doc.text
+    end
+  end
+end

data/lib/plaintext/file_handler.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class FileHandler
+    def accept?(content_type)
+      if @content_type
+        content_type == @content_type
+      elsif @content_types
+        @content_types.include? content_type
+      else
+        false
+      end
+    end
+  end
+end

data/lib/plaintext/resolver.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Plaintext
+  class Resolver
+    MAX_FULLTEXT_LENGTH = 4_194_304 # 4 megabytes
+
+    class << self
+      attr_accessor :cached_file_handlers
+
+      HANDLERS = [
+          Plaintext::PdfHandler,
+          Plaintext::OpendocumentHandler,
+          Plaintext::DocxHandler, Plaintext::XlsxHandler, Plaintext::PptxHandler,
+          Plaintext::DocHandler, Plaintext::XlsHandler, Plaintext::PptHandler,
+          Plaintext::ImageHandler,
+          Plaintext::RtfHandler,
+          Plaintext::PlaintextHandler
+      ].freeze
+
+      def file_handlers
+        return self.cached_file_handlers if self.cached_file_handlers.present?
+        self.cached_file_handlers = HANDLERS.map(&:new)
+      end
+    end
+
+    def initialize(file, content_type = nil)
+      @file = file
+      @content_type = content_type
+    end
+
+    # Returns the extracted fulltext or nil if no matching handler was found
+    # for the file type.
+    def text
+      if handler = find_handler and text = handler.text(@file)
+        text.gsub! /\s+/m, ' '
+        text.strip!
+        text.mb_chars.compose.limit(MAX_FULLTEXT_LENGTH).to_s
+      end
+    end
+
+    private
+
+    def find_handler
+      self.class.file_handlers.detect { |h| h.accept? @content_type }
+    end
+
+  end
+end

data/lib/plaintext/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Plaintext
+  VERSION = "0.1.0"
+end

data/lib/plaintext.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string'
+
+require 'plaintext/version'
+require 'plaintext/configuration'
+
+require 'plaintext/codeset_util'
+
+require 'plaintext/file_handler'
+require 'plaintext/file_handler/external_command_handler'
+require 'plaintext/file_handler/external_command_handler/doc_handler'
+require 'plaintext/file_handler/external_command_handler/image_handler'
+require 'plaintext/file_handler/external_command_handler/pdf_handler'
+require 'plaintext/file_handler/external_command_handler/ppt_handler'
+require 'plaintext/file_handler/external_command_handler/rtf_handler'
+require 'plaintext/file_handler/external_command_handler/xls_handler'
+
+require 'plaintext/file_handler/zipped_xml_handler'
+require 'plaintext/file_handler/zipped_xml_handler/office_document_handler'
+require 'plaintext/file_handler/zipped_xml_handler/office_document_handler/docx_handler'
+require 'plaintext/file_handler/zipped_xml_handler/office_document_handler/pptx_handler'
+require 'plaintext/file_handler/zipped_xml_handler/office_document_handler/xlsx_handler'
+require 'plaintext/file_handler/zipped_xml_handler/opendocument_handler'
+
+require 'plaintext/file_handler/plaintext_handler'
+
+require 'plaintext/resolver'

data/plaintext.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'plaintext/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "plaintext"
+  spec.version       = Plaintext::VERSION
+  spec.authors       = ['Jens Krämer', 'Planio GmbH', 'OpenProject GmbH']
+  spec.email         = ['info@openproject.com']
+
+  spec.summary       = 'Extract plain text from most common office documents.'
+  spec.description   = "Extract text from common office files. Based on the file's content type a command line tool is selected to do the job."
+  spec.homepage      = 'https://github.com/planio-gmbh/plaintext'
+  spec.license       = 'GPL-2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'rubyzip', '~> 1.2.1'
+  spec.add_dependency 'nokogiri', '~> 1.8.1'
+  spec.add_dependency 'activesupport', '>2.2.1 '
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

data/plaintext.yml.example

@@ -0,0 +1,41 @@
+# This is an example configuration file. Copy this file to your application config folder. In a Rails application that
+# would be `<Rails.root>/config/plaintext.yml`
+#
+# Text extraction helper programs.
+#
+# commands should write the resulting plain text to STDOUT. Use __FILE__ as
+# placeholder for the file path. The values below are the defaults.
+
+# apt-get install poppler-utils
+# pdftotext:
+#   - /usr/bin/pdftotext
+#   - -enc
+#   - UTF-8
+#   - __FILE__
+#   - '-'
+
+# apt-get install unrtf
+# unrtf:
+#   - /usr/bin/unrtf
+#   - --text
+#   - __FILE__
+
+# apt-get install catdoc
+# catdoc:
+#   - /usr/bin/catdoc
+#   - -dutf-8
+#   - __FILE__
+# xls2csv:
+#   - /usr/bin/xls2csv
+#   - -dutf-8
+#   - __FILE__
+# catppt:
+#   - /usr/bin/catppt
+#   - -dutf-8
+#   - __FILE__
+
+# apt-get install tesseract-ocr
+# tesseract:
+#   - /usr/bin/tesseract
+#   - -dutf-8
+#   - __FILE__

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: plaintext
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jens Krämer
+- Planio GmbH
+- OpenProject GmbH
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Extract text from common office files. Based on the file's content type
+  a command line tool is selected to do the job.
+email:
+- info@openproject.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/plaintext.rb
+- lib/plaintext/codeset_util.rb
+- lib/plaintext/configuration.rb
+- lib/plaintext/file_handler.rb
+- lib/plaintext/file_handler/external_command_handler.rb
+- lib/plaintext/file_handler/external_command_handler/doc_handler.rb
+- lib/plaintext/file_handler/external_command_handler/image_handler.rb
+- lib/plaintext/file_handler/external_command_handler/pdf_handler.rb
+- lib/plaintext/file_handler/external_command_handler/ppt_handler.rb
+- lib/plaintext/file_handler/external_command_handler/rtf_handler.rb
+- lib/plaintext/file_handler/external_command_handler/xls_handler.rb
+- lib/plaintext/file_handler/plaintext_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler/office_document_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/docx_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/pptx_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler/office_document_handler/xlsx_handler.rb
+- lib/plaintext/file_handler/zipped_xml_handler/opendocument_handler.rb
+- lib/plaintext/resolver.rb
+- lib/plaintext/version.rb
+- plaintext.gemspec
+- plaintext.yml.example
+homepage: https://github.com/planio-gmbh/plaintext
+licenses:
+- GPL-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Extract plain text from most common office documents.
+test_files: []