stevedore-uploader 1.0.7-java → 1.0.9-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 706101e4322d8e5bfd110b035115edf7bc02f971
-  data.tar.gz: 468209df9258209c9d86b28957a9e1ba71d13ffb
+  metadata.gz: d46b0968e5b625a0364cd9e165930b9f0ea0849b
+  data.tar.gz: 1f42e1aac685f31206f33fc6c7a68f06a94e8ab3
 SHA512:
-  metadata.gz: 2440f6a82e52a4a7746b347c2fdbac6fbaac429cc2ad2a8267b75919a61b15bbfa78b90c94188fcb67e0e322b4ea8d26be370e76827ae9c6c9d64f507a5cc7af
-  data.tar.gz: ab8ce1c340c94796a515c80f9cd66514daa5a415bdae2fd9efebe4f55a9b9932e302b963595bb7e108832bea3d379ac8d8eb4b98fd17dbe7aff887d1038f6fdf
+  metadata.gz: aa15c8bf07c5f1410396892e54b369dff1b5df14cf3521e02c7da449db84373f24c508bff3a9a28944741f8b4fb760505802b4975f62fb5aadffa516a29b6a5d
+  data.tar.gz: dd4c87fc77d0156dfca7324182d1f741e1babdf8320d499209d017b8b9061349f19220d5804b725225d9c794cbd2cde22bfb617a0d054fc025691368a69816b6

data/README.md

@@ -74,4 +74,4 @@ end
 Questions?
 ==========
 
-Hit us up in the [Stevedore](https://github.com/newsdev/stevedore) issues. Whichever suits your fancy.
+Hit us up in the [Stevedore](https://github.com/newsdev/stevedore) issues.

data/lib/parsers/stevedore_blob.rb

@@ -17,7 +17,7 @@ module Stevedore
     end 
 
     def self.new_from_tika(content, metadata, download_url, filename)
-      self.new(metadata["title"] || File.basename(filename), content, download_url)
+      self.new( ((metadata["title"] && metadata["title"] != "Untitled") ? metadata["title"] : File.basename(filename)), content, download_url)
     end
 
     def analyze!
@@ -26,12 +26,16 @@ module Stevedore
     end
 
     def to_hash
+      sha =  Digest::SHA1.hexdigest(download_url)
+      # TODO should merge in or something?
       {
-        "sha1" => Digest::SHA1.hexdigest(download_url),        
-        "title" => title.to_s,
+        "sha1" => sha,
+        "id" => sha,
+        "_id" => sha,
+        "title" => title.to_s || "Untitled Document: #{HumanHash::HumanHasher.new.humanize(sha)}",
         "source_url" => download_url.to_s,
         "file" => {
-          "title" => title.to_s,
+          "title" => title.to_s || "Untitled Document: #{HumanHash::HumanHasher.new.humanize(sha)}",
           "file" => clean_text.to_s
         },
         "analyzed" => {
@@ -40,7 +44,7 @@ module Stevedore
             "Content-Type" => extra["Content-Type"] || "text/plain"
           }
         },
-        "_updatedAt" => Time.now      
+        "_updatedAt" => Time.now,   
       }
     end
 

data/lib/parsers/stevedore_csv_row.rb

@@ -1,7 +1,7 @@
 require 'digest/sha1'
 
 module Stevedore
-  class StevedoreCsvRow
+  class StevedoreCsvRow < StevedoreBlob
     attr_accessor :title, :text, :download_url, :whole_row, :row_num
     def initialize(title, text, row_num, download_url, whole_row={})
       self.title = title || download_url

data/lib/parsers/stevedore_email.rb

@@ -2,12 +2,15 @@ require_relative './stevedore_blob'
 require 'cgi'
 require 'digest/sha1'
 require 'manticore'
+require 'dkimverify'
+
+
 module Stevedore
   class StevedoreEmail < StevedoreBlob
 
 
     # TODO write wrt other fields. where do those go???
-    attr_accessor :creation_date, :message_to, :message_from, :message_cc, :subject, :attachments, :content_type
+    attr_accessor :creation_date, :message_to, :message_from, :message_cc, :subject, :attachments, :content_type, :dkim_verified
 
     def self.new_from_tika(content, metadata, download_url, filepath)
       t = super
@@ -16,6 +19,11 @@ module Stevedore
       t.message_from = metadata["Message-From"]
       t.message_cc = metadata["Message-Cc"]
       t.title = t.subject = metadata["subject"]
+      t.dkim_verified = begin 
+                          Dkim::Verifier.new(filepath).verify!
+                        rescue Dkim::DkimError
+                          false
+                        end
       t.attachments = metadata["X-Attachments"].to_s.split("|").map do |raw_attachment_filename| 
         attachment_filename = CGI::unescape(raw_attachment_filename)
         possible_filename = File.join(File.dirname(filepath), attachment_filename)
@@ -72,7 +80,8 @@ module Stevedore
             "Message-From" => message_to.is_a?(Enumerable) ? message_to : [ message_to ],
             "Message-Cc" => message_cc.is_a?(Enumerable) ? message_cc : [ message_cc ],
             "subject" => subject,
-            "attachments" => attachments
+            "attachments" => attachments,
+            "dkim_verified" => dkim_verified
           }
         },
         "_updatedAt" => Time.now

data/lib/stevedore-uploader.rb

@@ -138,7 +138,7 @@ module Stevedore
         # TODO: factor these out in favor of the yield/block situation down below.
         # this should (eventually) be totally generic, but perhaps handle common 
         # document types on its own
-        ret = case                             # .eml                                          # .msg
+        doc = case                             # .eml                                          # .msg
               when metadata["Content-Type"] == "message/rfc822" || metadata["Content-Type"] == "application/vnd.ms-outlook"
                 ::Stevedore::StevedoreEmail.new_from_tika(content, metadata, download_url, filename).to_hash
               when metadata["Content-Type"] && ["application/html", "application/xhtml+xml"].include?(metadata["Content-Type"].split(";").first)
@@ -156,16 +156,19 @@ module Stevedore
                   File.delete("#{png}.txt") rescue nil
                 end.join("\n\n")
                 # e.g.  Analysis-Corporation-2.png.pdf or Torture.pdf
-                files = Dir["#{pdf_basename}.png.pdf"] + (Dir["#{pdf_basename}-*.png.pdf"].sort_by{|pdf| Regexp.new("#{pdf_basename}-([0-9]+).png.pdf").match(pdf)[1].to_i })
-                return nil if files.empty?
-                system('pdftk', *files, "cat", "output", "#{pdf_basename}.ocr.pdf")
-                content, _ = Rika.parse_content_and_metadata("#{pdf_basename}.ocr.pdf")
+                files = Dir["#{pdf_basename}.png.pdf"] + (Dir["#{pdf_basename}-*.png.pdf"].sort_by{|pdf| (m = Regexp.new("#{pdf_basename}-([0-9]+).png.pdf").match(pdf)) ? m[1].to_i : 69420 }) # 69420 is a random really big number, sorting those docs to the end.
+                if files.empty?
+                  content = ''
+                else
+                  system('pdftk', *files, "cat", "output", "#{pdf_basename}.ocr.pdf")
+                  content, _ = Rika.parse_content_and_metadata("#{pdf_basename}.ocr.pdf")
+                end
                 puts "OCRed content (#{File.basename(filename)}) length: #{content.length}"
                 ::Stevedore::StevedoreBlob.new_from_tika(content, metadata, download_url, filename).to_hash
               else
                 ::Stevedore::StevedoreBlob.new_from_tika(content, metadata, download_url, filename).to_hash
               end
-      [ret, content, metadata]
+      [doc, content, metadata]
       rescue StandardError, java.lang.NoClassDefFoundError, org.apache.tika.exception.TikaException => e
         STDERR.puts e.inspect
         STDERR.puts "#{e} #{e.message}: #{filename}"
@@ -220,7 +223,6 @@ module Stevedore
           s3_path_without_bucket = target_path.gsub(/s3:\/\//i, '').split("/", 2).last
           bucket.objects(:prefix => s3_path_without_bucket).each_slice(@slice_size) do |slice_of_objs|
             docs_so_far += slice_of_objs.size
-
             output_stream.puts "starting a set of #{@slice_size} -- so far #{docs_so_far}"
             slice_of_objs.map! do |obj|
               next if obj.key[-1] == "/"
@@ -244,12 +246,19 @@ module Stevedore
               if ArchiveSplitter::HANDLED_FORMATS.include?(tmp_filename.split(".")[-1]) 
                 ArchiveSplitter.split(tmp_filename).map do |constituent_file, constituent_basename, attachment_basenames, parent_basename|
                   doc, content, metadata = process_document(constituent_file, download_filename)
+                  next nil if doc.nil?
                   doc["analyzed"] ||= {}
                   doc["analyzed"]["metadata"] ||= {}
+
+                  # this is a hack: but we're replicating how IDs are calculated (in parsers/stevedore_blob.rb) to make "attachments" the list of IDs of all documents in the archive
+                  # we have to set separate sha1s for these, because they're by default based only on the download URL (which is the same for all of the constituent files)
                   doc["analyzed"]["metadata"]["attachments"] = (parent_basename.nil? ? [] : [Digest::SHA1.hexdigest(download_filename + parent_basename)]) + attachment_basenames.map{|attachment| Digest::SHA1.hexdigest(download_filename + attachment) } # is a list of filenames
                   doc["sha1"] = Digest::SHA1.hexdigest(download_filename + File.basename(constituent_basename)) # since these files all share a download URL (that of the archive, we need to come up with a custom sha1)
+                  doc["id"] = doc["sha1"]
+                  doc["_id"] = doc["sha1"]
                   yield doc, obj.key, content, metadata if block_given?
                   FileUtils.rm(constituent_file) rescue Errno::ENOENT # try to delete, but no biggie if it doesn't work for some weird reason.
+                  doc["file"]["title"] ||= "Untitled Document: #{HumanHash::HumanHasher.new.humanize(doc["_id"])}"
                   doc
                 end
               else 
@@ -300,14 +309,20 @@ module Stevedore
             if ArchiveSplitter::HANDLED_FORMATS.include?(filename.split(".")[-1]) 
                 ArchiveSplitter.split(filename).map do |constituent_file, constituent_basename, attachment_basenames, parent_basename|
                 doc, content, metadata = process_document(constituent_file, download_filename)
-                doc = {} if doc.nil?
+                next nil if doc.nil?
                 doc["analyzed"] ||= {}
                 doc["analyzed"]["metadata"] ||= {}
+                
+                # this is a hack: but we're replicating how IDs are calculated (in parsers/stevedore_blob.rb) to make "attachments" the list of IDs of all documents in the archive
+                # we have to set separate sha1s for these, because they're by default based only on the download URL (which is the same for all of the constituent files)
                 doc["analyzed"]["metadata"]["attachments"] = (parent_basename.nil? ? [] : [Digest::SHA1.hexdigest(download_filename + parent_basename)]) + attachment_basenames.map{|attachment| Digest::SHA1.hexdigest(download_filename + attachment) } # is a list of filenames
                 doc["sha1"] = Digest::SHA1.hexdigest(download_filename + File.basename(constituent_basename)) # since these files all share a download URL (that of the archive, we need to come up with a custom sha1)
                 doc["id"] = doc["sha1"]
+                doc["_id"] = doc["sha1"]
+                
                 yield doc, filename, content, metadata if block_given?
                 # FileUtils.rm(constituent_file) rescue Errno::ENOENT # try to delete, but no biggie if it doesn't work for some weird reason.
+                puts doc.inspect
                 doc
               end
             else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stevedore-uploader
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.9
 platform: java
 authors:
 - Jeremy B. Merrill
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: '1.7'
   name: rika-stevedore
   prerelease: false
   type: :runtime
@@ -79,7 +79,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -136,6 +136,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: humanhash
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  name: dkimverify
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 description: TK
 email: jeremy.merrill@nytimes.com
 executables: []
@@ -170,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.6
 signing_key:
 specification_version: 4
 summary: Upload documents to a Stevedore search engine.