neon_documents 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ac10ee5c8b168a8c0962a03575afa4f2354eb2cc841f177ef406d52b4b13308d
+  data.tar.gz: 794195193edefa7b7203a819fe88c0cdad46de2cbab605c26f558df54e4d38ab
+SHA512:
+  metadata.gz: 4344aaaa9f767494b1b13b4086e4dfdf70a8639cf01a6ab6e5c6762258ec6049c79160ee828794d7fa598cab1de4b911d2edc57387ade28084f544220894ac83
+  data.tar.gz: df0996a61516f0f9a5df3275e410a30e38294f853de80470efecae51d3d898d9b7f10cf4bede3d0e8cd23a25557a6c0cf44144007bbd8cb16fe4168a9399a581

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,22 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in neon_postgres.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+gem "standard"
+
+gem "sequel", "~> 5.43"
+gem "fixture_dependencies", "~> 1.10"
+gem "pg", "~> 1.2"
+
+gem "pry", "~> 0.14.1"
+
+gem "faker", "~> 2.17"
+
+gem "ruby-kafka", "~> 1.3"
+
+gem "sendgrid-ruby", "~> 6.4"
+
+gem "neon_postgres", "~> 0.0.1"

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    neon_documents (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.8)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.3)
+      rake (>= 12.0.0, < 14.0.0)
+    faker (2.17.0)
+      i18n (>= 1.6, < 2)
+    fixture_dependencies (1.10.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    neon_postgres (0.0.1)
+    parallel (1.20.1)
+    parser (3.0.1.0)
+      ast (~> 2.4.1)
+    pg (1.2.3)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.12.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.10.1)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-kafka (1.3.0)
+      digest-crc
+    ruby-progressbar (1.11.0)
+    ruby_http_client (3.5.2)
+    sendgrid-ruby (6.4.0)
+      ruby_http_client (~> 3.4)
+    sequel (5.43.0)
+    standard (1.0.5)
+      rubocop (= 1.12.1)
+      rubocop-performance (= 1.10.1)
+    unicode-display_width (2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  faker (~> 2.17)
+  fixture_dependencies (~> 1.10)
+  neon_documents!
+  neon_postgres (~> 0.0.1)
+  pg (~> 1.2)
+  pry (~> 0.14.1)
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  ruby-kafka (~> 1.3)
+  sendgrid-ruby (~> 6.4)
+  sequel (~> 5.43)
+  standard
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -0,0 +1,18 @@
+# Neon Documents
+
+This gem is used to listen to document messages in Kafka and then interact with
+the GCP Storage API.
+
+## Necessary API Keys and environment variables.
+
+* Kafka URL - Where to consume document message
+* Postgres URL - The PostgreSQL URL for connecting
+* GCP Credentials - A GCP Key JSON for a service account with access to the
+  UNPROCESSED_DOCUMENTS_BUCKET and the PRIVATE_DOCUMENT_BUCKET
+* UNPROCESSED_DOCUMENTS_BUCKET - the GCP bucket name for unprocessed documents
+* PRIVATE_DOCUMENT_BUCKET - the GCP bucket for processed, private documents
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/neonlaw/codebase.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/lib/neon_documents.rb

@@ -0,0 +1,10 @@
+require "neon_documents/version"
+require "neon_documents/consumer"
+require "neon_documents/document_processor"
+Dir["#{__dir__}/neon_documents/document_templates/*"].sort.each { |file|
+  require file
+}
+
+module NeonDocuments
+  class Error < StandardError; end
+end

data/lib/neon_documents/consumer.rb

@@ -0,0 +1,30 @@
+require "kafka"
+
+module NeonDocuments
+  class Consumer
+    def self.consume_messages
+      kafka = Kafka.new([ENV.fetch("KAFKA_URL") { "kafka:9092" }])
+
+      # Consumers with the same group id will form a Consumer Group together.
+      consumer = kafka.consumer(group_id: "documents")
+
+      # It's possible to subscribe to multiple topics by calling `subscribe`
+      # repeatedly.
+      consumer.subscribe("process_document")
+
+      # Stop the consumer when the SIGTERM signal is sent to the process.
+      # It's better to shut down gracefully than to kill the process.
+      trap("TERM") { consumer.stop }
+
+      # This will loop indefinitely, yielding each message in turn.
+      consumer.each_message do |message|
+        case message.topic
+        when "process_document"
+          NeonDocument::DocumentProcessor.encode_and_store_document(
+            unprocessed_document_id: message.value
+          )
+        end
+      end
+    end
+  end
+end

data/lib/neon_documents/document_processor.rb

@@ -0,0 +1,46 @@
+require "sendgrid-ruby"
+
+module NeonDocuments
+  class DocumentProcessor
+    def self.encode_and_store_document(unprocessed_document_id:)
+      new(
+        unprocessed_document_id: unprocessed_document_id
+      ).encode_and_store_document
+    end
+
+    def initialize(unprocessed_document_id:)
+      @unprocessed_document_id = unprunprocessed_document_id
+    end
+
+    def encode_and_store_document
+      filename = case document_template.name
+      when "Webpage Screenshot"
+        NeonDocuments::DocumentTemplates::WebpageScreenshot.processed_filename(
+          unprocessed_filename: unprocessed_document.filename
+        )
+      end
+
+      puts filename
+    end
+
+    private
+
+    attr_reader :unprocessed_document_id
+
+    def connection
+      @_connection ||= NeonPostgres::Database.connection
+    end
+
+    def unprocessed_document
+      @_unprocessed_document ||= connection[:unprocessed_documents].find(
+        id: unprocessed_document_id
+      )
+    end
+
+    def document_template
+      @_document_template ||= connection[:document_templates].find(
+        id: unprocessed_document.fetch(:document_template_id)
+      )
+    end
+  end
+end

data/lib/neon_documents/document_templates/webpage_screenshot.rb

@@ -0,0 +1,11 @@
+module NeonDocuments
+  module DocumentTemplates
+    class WebpageScreenshot
+      def self.processed_filename(unprocessed_filename:)
+        unprocessed_filename
+          .gsub(/:\/\//, "___")
+          .tr(".", "_")
+      end
+    end
+  end
+end

data/lib/neon_documents/version.rb

@@ -0,0 +1,3 @@
+module NeonDocuments
+  VERSION = "0.0.1"
+end

data/neon_documents.gemspec

@@ -0,0 +1,25 @@
+require_relative "lib/neon_documents/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "neon_documents"
+  spec.version = NeonDocuments::VERSION
+  spec.authors = ["Neon Law"]
+  spec.email = ["support@neonlaw.com"]
+
+  spec.summary = "A gem to process files."
+  spec.description = 'This gem contains tests and sequel setup to speak to
+  the Neon Law database'
+  spec.homepage = "https://github.com/neonlaw/codebase"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: neon_documents
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Neon Law
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-05-10 00:00:00.000000000 Z
+dependencies: []
+description: |-
+  This gem contains tests and sequel setup to speak to
+    the Neon Law database
+email:
+- support@neonlaw.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/neon_documents.rb
+- lib/neon_documents/consumer.rb
+- lib/neon_documents/document_processor.rb
+- lib/neon_documents/document_templates/webpage_screenshot.rb
+- lib/neon_documents/version.rb
+- neon_documents.gemspec
+homepage: https://github.com/neonlaw/codebase
+licenses: []
+metadata:
+  homepage_uri: https://github.com/neonlaw/codebase
+  source_code_uri: https://github.com/neonlaw/codebase
+  changelog_uri: https://github.com/neonlaw/codebase
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: A gem to process files.
+test_files: []