chupa-text 1.1.5 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65f2eb5be135f3db4a0cbb81bc4a268bcb752934da6d9aa32740842bb4cc674f
-  data.tar.gz: 72f8487dc57b67293e001621f6b63ece4680cf1069d9f98e3daa89708571766b
+  metadata.gz: 8a248b3b82aeb4ce65bd7b498dc2aab9ad6e43e6d5ea70a304163362c3e723e7
+  data.tar.gz: 82f3f02b63235924cab8608f1fde8d4ecbe4cb0e1aa92f12bf9ecff1debab21d
 SHA512:
-  metadata.gz: 32149617d9c921de856aa9fb3856a8a29fde5e9e1872a16241235499b1b8e014941928fb040ad60d52c15cbc4b69d84a9860bd432074964865e7f1e85d673a1e
-  data.tar.gz: 85c2820468547c9b34fef27faa2db0fc9d8208a266949177b2e4f2269bfadcd8f5b534c0cdb4aff9e28360c1509dd6b2d06ef72c7d1e268f399e97595a133a2c
+  metadata.gz: f2c3acee194db640b3b3e01a913cfc00727ed7b8e0e4b353f5fbb2d1723f689846164d44f2ffcf1874b2948ec3f3b578dacd49d27934693ad71f24b9901373a2
+  data.tar.gz: 12bd3c5ed94f85bad888e56a62ef478a606e8ce5fe8f376f7c2418924ef91094b1a6c150c60fd80278a7541bb5e177b7d9cb4b7ec5b67a482b8ff74c67953ba2

data/chupa-text.gemspec

@@ -50,7 +50,8 @@ Gem::Specification.new do |spec|
     spec.executables = Dir.glob("*")
   end
 
-  spec.add_runtime_dependency("archive-zip")
+  spec.add_runtime_dependency("archive-zip", ">= 0.12.0")
+  spec.add_runtime_dependency("csv", ">= 3.0.4")
 
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("nokogiri")

data/doc/text/news.md

@@ -1,5 +1,47 @@
 # News
 
+## 1.1.6: 2019-03-01
+
+### Improvements
+
+  * `zip`:
+
+    * Added support for multibyte path.
+
+    * Added error check.
+
+  * `tar`:
+
+    * Added support for multibyte path.
+
+    * Reduced memory usage.
+
+  * Changed to the extracted text encoding to UTF-8.
+
+  * Added support BOM detection.
+
+  * Improved binary data detection.
+
+  * `office-open-xml-workbook`:
+
+    * Added support for not shared string cell values.
+
+    * Changed to emit data per sheet.
+
+  * `office-open-xml-presentation`:
+
+    * Changed to emit data per slide.
+
+  * `csv`:
+
+    * Added error check.
+
+  * `opendocument-spreadsheet`:
+
+    * Added support for concatenated cell.
+
+    * Added support for shapes.
+
 ## 1.1.5: 2019-02-28
 
 ### Improvements

data/lib/chupa-text/data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,6 +18,8 @@ require "cgi/util"
 require "uri"
 require "open-uri"
 
+require "chupa-text/utf8-converter"
+
 module ChupaText
   class Data
     # @return [URI, nil] The URI of the data if the data is for remote
@@ -190,6 +192,18 @@ module ChupaText
       @need_screenshot
     end
 
+    def to_utf8_body_data
+      b = body
+      return self if b.nil?
+      converter = UTF8Converter.new(b)
+      utf8_body = converter.convert
+      if b.equal?(utf8_body)
+        self
+      else
+        TextData.new(utf8_body, source_data: self)
+      end
+    end
+
     private
     def guess_mime_type
       guess_mime_type_from_uri or
@@ -203,7 +217,10 @@ module ChupaText
     def guess_mime_type_from_body
       mime_type = nil
       change_encoding(body, "UTF-8") do |utf8_body|
-        mime_type = "text/plain" if utf8_body.valid_encoding?
+        return nil unless utf8_body.valid_encoding?
+        n_null_characters = utf8_body.count("\u0000")
+        return nil if n_null_characters > (utf8_body.bytesize * 0.01)
+        mime_type = "text/plain"
       end
       mime_type
     end

data/lib/chupa-text/decomposers/csv.rb

@@ -20,6 +20,8 @@ require "csv"
 module ChupaText
   module Decomposers
     class CSV < Decomposer
+      include Loggable
+
       registry.register("csv", self)
 
       def target?(data)
@@ -36,10 +38,20 @@ module ChupaText
       def decompose(data)
         text = ""
         data.open do |input|
-          csv = ::CSV.new(input)
-          csv.each do |row|
-            text << row.join(" ")
-            text << "\n"
+          begin
+            csv = ::CSV.new(input, liberal_parsing: true)
+            csv.each do |row|
+              text << row.join("\t")
+              text << "\n"
+            end
+          rescue ::CSV::MalformedCSVError => csv_error
+            error do
+              message = "#{log_tag} Failed to parse CSV: "
+              message << "#{csv_error.class}: #{csv_error.message}\n"
+              message << csv_error.backtrace.join("\n")
+              message
+            end
+            return
           end
         end
 
@@ -78,6 +90,10 @@ module ChupaText
         SVG
         Screenshot.new(mime_type, data)
       end
+
+      def log_tag
+        "[decomposer][csv]"
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/office-open-xml-document.rb

@@ -40,12 +40,24 @@ module ChupaText
       end
 
       private
+      def start_decompose(context)
+        context[:text] = ""
+      end
+
       def process_entry(entry, context)
         case entry.zip_path
         when "word/document.xml"
           extract_text(entry, context[:text])
         end
       end
+
+      def finish_decompose(context, &block)
+        text_data = TextData.new(context[:text], source_data: context[:data])
+        context[:attributes].each do |name, value|
+          text_data[name] = value
+        end
+        yield(text_data)
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/office-open-xml-presentation.rb

@@ -48,19 +48,33 @@ module ChupaText
       end
 
       private
+      def start_decompose(context)
+        context[:slides] = []
+      end
+
       def process_entry(entry, context)
         case entry.zip_path
         when /\Appt\/slides\/slide(\d+)\.xml/
           nth_slide = Integer($1, 10)
           slide_text = ""
           extract_text(entry, slide_text)
-          context[:slides] ||= []
           context[:slides] << [nth_slide, slide_text]
         end
       end
 
-      def accumulate_text(context)
-        context[:slides].sort_by(&:first).collect(&:last).join("\n")
+      def finish_decompose(context, &block)
+        metadata = TextData.new("", source_data: context[:data])
+        context[:attributes].each do |name, value|
+          metadata[name] = value
+        end
+        yield(metadata)
+
+        slide_texts = context[:slides].sort_by(&:first).collect(&:last)
+        slide_texts.each_with_index do |slide_text, i|
+          text_data = TextData.new(slide_text, source_data: context[:data])
+          text_data["index"] = i
+          yield(text_data)
+        end
       end
     end
   end

data/lib/chupa-text/decomposers/office-open-xml-workbook.rb

@@ -40,35 +40,73 @@ module ChupaText
       end
 
       private
+      def start_decompose(context)
+        context[:shared_strings] = []
+        context[:sheet_names] = []
+        context[:sheets] = []
+      end
+
       def process_entry(entry, context)
         case entry.zip_path
         when "xl/sharedStrings.xml"
-          context[:shared_strings] = []
           extract_text(entry, context[:shared_strings])
+        when "xl/workbook.xml"
+          listener = WorkbookListener.new(context[:sheet_names])
+          parse(entry.file_data, listener)
         when /\Axl\/worksheets\/sheet(\d+)\.xml\z/
           nth_sheet = Integer($1, 10)
           sheet = []
           listener = SheetListener.new(sheet)
           parse(entry.file_data, listener)
-          context[:sheets] ||= []
           context[:sheets] << [nth_sheet, sheet]
         end
       end
 
-      def accumulate_text(context)
+      def finish_decompose(context, &block)
+        metadata = TextData.new("", source_data: context[:data])
+        context[:attributes].each do |name, value|
+          metadata[name] = value
+        end
+        yield(metadata)
+
         shared_strings = context[:shared_strings]
         sheets = context[:sheets].sort_by(&:first).collect(&:last)
-        sheet_texts = sheets.collect do |sheet|
+        sheet_names = context[:sheet_names]
+        sheets.each_with_index do |sheet, i|
           sheet_text = ""
           sheet.each do |row|
-            row_texts = row.collect do |index|
-              shared_strings[index]
+            row_texts = row.collect do |cell|
+              case cell
+              when Integer
+                shared_strings[cell]
+              else
+                cell
+              end
             end
             sheet_text << row_texts.join("\t") << "\n"
           end
-          sheet_text
+          text_data = TextData.new(sheet_text, source_data: context[:data])
+          text_data["index"] = i
+          name = sheet_names[i]
+          text_data["name"] = name if name
+          yield(text_data)
+        end
+      end
+
+      class WorkbookListener < SAXListener
+        URI = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+
+        def initialize(sheet_names)
+          @sheet_names = sheet_names
+        end
+
+        def start_element(uri, local_name, qname, attributes)
+          return unless uri == URI
+          case local_name
+          when "sheet"
+            @sheet_names << attributes["name"]
+          end
         end
-        sheet_texts.join("\n")
       end
 
       class SheetListener < SAXListener
@@ -76,6 +114,7 @@ module ChupaText
 
         def initialize(sheet)
           @sheet = sheet
+          @cell_type = nil
           @in_v = false
         end
 
@@ -84,13 +123,22 @@ module ChupaText
           case local_name
           when "row"
             @sheet << []
+          when "c"
+            @cell_type = parse_cell_type(attributes["t"])
+          # when "is" # TODO
           when "v"
             @in_v = true
           end
         end
 
         def end_element(uri, local_name, qname)
-          @in_v = false
+          return unless uri == URI
+          case local_name
+          when "c"
+            @cell_type = nil
+          when "v"
+            @in_v = false
+          end
         end
 
         def characters(text)
@@ -102,9 +150,34 @@ module ChupaText
         end
 
         private
+        # https://c-rex.net/projects/samples/ooxml/e1/Part4/OOXML_P4_DOCX_ST_CellType_topic_ID0E6NEFB.html
+        def parse_cell_type(type)
+          case type
+          when "b"
+            :boolean
+          when "e"
+            :error
+          when "inlineStr"
+            :inline_string
+          when "n"
+            :number
+          when "s"
+            :shared_string
+          when "str"
+            :string
+          else
+            nil
+          end
+        end
+
         def add_column(text)
           return unless @in_v
-          @sheet.last << Integer(text, 10)
+          case @cell_type
+          when :shared_string
+            @sheet.last << Integer(text, 10)
+          else
+            @sheet.last << text
+          end
         end
       end
     end

data/lib/chupa-text/decomposers/office-open-xml.rb

@@ -34,11 +34,12 @@ module ChupaText
         end
       end
 
-      def decompose(data)
+      def decompose(data, &block)
         context = {
-          text: "",
+          data: data,
           attributes: {},
         }
+        start_decompose(context)
         data.open do |input|
           Archive::Zip.open(input) do |zip|
             zip.each do |entry|
@@ -56,12 +57,7 @@ module ChupaText
             end
           end
         end
-        text = accumulate_text(context)
-        text_data = TextData.new(text, source_data: data)
-        context[:attributes].each do |name, value|
-          text_data[name] = value
-        end
-        yield(text_data)
+        finish_decompose(context, &block)
       end
 
       private

data/lib/chupa-text/decomposers/opendocument-spreadsheet.rb

@@ -60,6 +60,7 @@ module ChupaText
           @prefix_to_uri = {}
           @uri_to_prefix = {}
           @in_p = false
+          @in_shapes = false
         end
 
         def start_prefix_mapping(prefix, uri)
@@ -86,29 +87,44 @@ module ChupaText
               @sheets << {
                 name: attributes["#{table_prefix}:name"],
                 rows: [],
+                shape_texts: [],
               }
             when "table-row"
               @sheets.last[:rows] << []
             when "table-cell"
               @sheets.last[:rows].last << {text: ""}
+            when "covered-table-cell"
+              @sheets.last[:rows].last << {text: ""}
+            when "shapes"
+              @in_shapes = true
             end
           end
         end
 
         def end_element(uri, local_name, qname)
-          @in_p = false
           case uri
+          when TEXT_URI
+            case local_name
+            when "p"
+              @in_p = false
+            end
           when TABLE_URI
             case local_name
             when "table"
               sheet = @sheets.last
               text = ""
+              shape_texts = sheet[:shape_texts]
+              unless shape_texts.empty?
+                text << shape_texts.join("\n") << "\n"
+              end
               sheet[:rows].each do |row|
                 cell_texts = row.collect {|cell| cell[:text]}
                 next if cell_texts.all?(&:empty?)
                 text << cell_texts.join("\t") << "\n"
               end
               sheet[:text] = text
+            when "shapes"
+              @in_shapes = false
             end
           end
         end
@@ -124,7 +140,15 @@ module ChupaText
         private
         def add_text(text)
           return unless @in_p
-          @sheets.last[:rows].last.last[:text] << text
+          sheet = @sheets.last
+          if @in_shapes
+            sheet[:shape_texts] << text
+          else
+            sheet[:rows].last.last[:text] << text
+          end
+        rescue
+          pp [text, @sheets]
+          raise
         end
       end
     end