chronicle-etl 0.1.1 → 0.2.1

data/lib/chronicle/etl/cli/main.rb

@@ -0,0 +1,83 @@
+require 'thor'
+require 'chronicle/etl'
+require 'colorize'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+
+module Chronicle
+  module ETL
+    module CLI
+      # Main entrypoint for CLI app
+      class Main < Thor
+        class_option "verbose", type: :boolean, default: false
+        default_task "jobs"
+
+        desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
+        subcommand 'connectors', Connectors
+
+        desc 'jobs:COMMAND', 'Configure and run jobs', hide: true
+        subcommand 'jobs', Jobs
+
+        # Entrypoint for the CLI
+        def self.start(given_args = ARGV, config = {})
+          if given_args.none?
+            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
+          end
+
+          # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
+          if given_args.any? && given_args[0].include?(':')
+            commands = given_args.shift.split(':')
+            given_args = given_args.unshift(commands).flatten
+          end
+
+          super(given_args, config)
+        end
+
+        # Displays help options for chronicle-etl
+        def help(meth = nil, subcommand = false)
+          if meth && !respond_to?(meth)
+            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass.start(['-h', task].compact, shell: shell)
+          else
+            shell.say "ABOUT".bold
+            shell.say "  #{'chronicle-etl'.italic} is a utility tool for #{'extracting'.underline}, #{'transforming'.underline}, and #{'loading'.underline} personal data."
+            shell.say
+            shell.say "USAGE".bold
+            shell.say "  $ chronicle-etl COMMAND"
+            shell.say
+            shell.say "EXAMPLES".bold
+            shell.say "  Show available connectors:".italic.light_black
+            shell.say "  $ chronicle-etl connectors:list"
+            shell.say
+            shell.say "  Run a simple job:".italic.light_black
+            shell.say "  $ chronicle-etl jobs:start --extractor stdin --transformer null --loader stdout"
+            shell.say
+            shell.say "  Show full job options:".italic.light_black
+            shell.say "  $ chronicle-etl jobs help start"
+
+            list = []
+
+            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+              list += thor_class.printable_tasks(false)
+            end
+            list.sort! { |a, b| a[0] <=> b[0] }
+            list.unshift ["help", "# This help menu"]
+
+            shell.say
+            shell.say 'ALL COMMANDS'.bold
+            shell.print_table(list, indent: 2, truncate: true)
+            shell.say
+            shell.say "VERSION".bold
+            shell.say "  #{Chronicle::ETL::VERSION}"
+            shell.say
+            shell.say "FULL DOCUMENTATION".bold
+            shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
+            shell.say
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -0,0 +1,37 @@
+module Chronicle
+  module ETL
+    module CLI
+      # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
+      class SubcommandBase < Thor
+        # Print usage instructions for a subcommand
+        def self.help(shell, subcommand = false)
+          list = printable_commands(true, subcommand)
+          Thor::Util.thor_classes_in(self).each do |klass|
+            list += klass.printable_commands(false)
+          end
+          list.sort! { |a, b| a[0] <=> b[0] }
+
+          shell.say "COMMANDS".bold
+          shell.print_table(list, indent: 2, truncate: true)
+          shell.say
+          class_options_help(shell)
+        end
+
+        # Show docs with command:subcommand pattern.
+        # For `help` command, don't use colon
+        def self.banner(command, namespace = nil, subcommand = false)
+          if command.name == 'help'
+            "#{subcommand_prefix} #{command.usage}"
+          else
+            "#{subcommand_prefix}:#{command.usage}"
+          end
+        end
+
+        # Use subcommand classname to derive display name for subcommand
+        def self.subcommand_prefix
+          self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/config.rb

@@ -0,0 +1,32 @@
+require 'runcom'
+
+module Chronicle
+  module ETL
+    # Utility methods to read, write, and access config files
+    module Config
+      # Loads a yml config file
+      def self.load(path)
+        config = Runcom::Config.new(path)
+        # FIXME: hack to deeply symbolize keys
+        JSON.parse(config.to_h.to_json, symbolize_names: true)
+      end
+
+      # Writes a hash as a yml config file
+      def self.write(path, data)
+        config = Runcom::Config.new(path)
+        filename = config.all[0].to_s + '.yml'
+        File.open(filename, 'w') do |f|
+          f << data.to_yaml
+        end
+      end
+
+      # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
+      def self.jobs
+        job_directory = Runcom::Config.new('chronicle/etl/jobs').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/{csv.rb → csv_extractor.rb}

@@ -1,5 +1,5 @@
 require 'csv'
-class Chronicle::Etl::Extractors::Csv < Chronicle::Etl::Extractors::Extractor
+class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
   DEFAULT_OPTIONS = {
     headers: true,
     filename: $stdin
@@ -18,7 +18,7 @@ class Chronicle::Etl::Extractors::Csv < Chronicle::Etl::Extractors::Extractor
   end
 
   def results_count
-    CSV.read(@options[:filename], { headers: @options[:headers] }).count if read_from_file?
+    CSV.read(@options[:filename], headers: @options[:headers]).count if read_from_file?
   end
 
   private
@@ -33,7 +33,7 @@ class Chronicle::Etl::Extractors::Csv < Chronicle::Etl::Extractors::Extractor
     }
 
     stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]
-    CSV.new(stream, csv_options)
+    CSV.new(stream, **csv_options)
   end
 
   def read_from_file?

data/lib/chronicle/etl/extractors/extractor.rb

@@ -1,20 +1,31 @@
+require 'chronicle/etl'
+
 module Chronicle
-  module Etl
-    module Extractors
-      class Extractor
-        def initialize(options = {})
-          @options = options.transform_keys!(&:to_sym) 
-        end
+  module ETL
+    # Abstract class representing an Extractor for an ETL job
+    class Extractor
+      extend Chronicle::ETL::Catalog
 
-        def extract
-          raise NotImplementedError
-        end
+      # Construct a new instance of this extractor. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Extractor
+      def initialize(options = {})
+        @options = options.transform_keys!(&:to_sym)
+      end
 
-        def results_count; end
+      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
+      def extract
+        raise NotImplementedError
       end
+
+      # An optional method to calculate how many records there are to extract. Used primarily for
+      # building the progress bar
+      def results_count; end
     end
   end
 end
 
-require_relative 'stdin'
-require_relative 'csv'
+require_relative 'csv_extractor'
+require_relative 'file_extractor'
+require_relative 'stdin_extractor'

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -0,0 +1,52 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    class FileExtractor < Chronicle::ETL::Extractor
+      def extract
+        if file?
+          extract_file do |data, metadata|
+            yield(data, metadata)
+          end
+        elsif directory?
+          extract_from_directory do |data, metadata|
+            yield(data, metadata)
+          end
+        end
+      end
+
+      def results_count
+        if file?
+          return 1
+        else
+          search_pattern = File.join(@options[:filename], '**/*.eml')
+          Dir.glob(search_pattern).count
+        end
+      end
+
+      private
+
+      def extract_from_directory
+        search_pattern = File.join(@options[:filename], '**/*.eml')
+        filenames = Dir.glob(search_pattern)
+        filenames.each do |filename|
+          file = File.open(filename)
+          yield(file.read, {filename: file})
+        end
+      end
+
+      def extract_file
+        file = File.open(@options[:filename])
+        yield(file.read, {filename: @options[:filename]})
+      end
+
+      def directory?
+        Pathname.new(@options[:filename]).directory?
+      end
+
+      def file?
+        Pathname.new(@options[:filename]).file?
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -0,0 +1,11 @@
+module Chronicle
+  module ETL
+    class StdinExtractor < Chronicle::ETL::Extractor
+      def extract
+        $stdin.read.each_line do |line|
+          yield line
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -0,0 +1,29 @@
+require 'csv'
+
+module Chronicle
+  module ETL
+    class CsvLoader < Chronicle::ETL::Loader
+      def initialize(options={})
+        super(options)
+        @rows = []
+      end
+
+      def load(result)
+        if (result.is_a? Hash)
+          @rows << result.values
+        else
+          @rows << result
+        end
+      end
+
+      def finish
+        z = $stdout
+        CSV(z) do |csv|
+          @rows.each do |row|
+            csv << row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,25 +1,32 @@
 module Chronicle
-  module Etl
-    module Loaders
-      class Loader
-        def initialize(options = {})
-          @options = options
-        end
+  module ETL
+    # Abstract class representing a Loader for an ETL job
+    class Loader
+      extend Chronicle::ETL::Catalog
 
-        def start; end
-
-        def first_load; end
+      # Construct a new instance of this loader. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Loader
+      def initialize(options = {})
+        @options = options
+      end
 
-        def load
-          raise NotImplementedError
-        end
+      # Called once before processing records
+      def start; end
 
-        def finish; end
+      # Load a single record
+      def load
+        raise NotImplementedError
       end
+
+      # Called once there are no more records to process
+      def finish; end
     end
   end
 end
 
-require_relative 'stdout'
-require_relative 'csv'
-require_relative 'table'
+require_relative 'csv_loader'
+require_relative 'rest_loader'
+require_relative 'stdout_loader'
+require_relative 'table_loader'

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -0,0 +1,30 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Chronicle
+  module ETL
+    class RestLoader < Chronicle::ETL::Loader
+      def initialize(options={})
+        super(options)
+      end
+
+      def load(result)
+        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+
+        header = {
+          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Content-Type": 'application/json'
+        }
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        request = Net::HTTP::Post.new(uri.request_uri, header)
+
+        obj = {data: result} unless result[:data]
+        request.body = obj.to_json
+
+        response = http.request(request)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -0,0 +1,9 @@
+module Chronicle
+  module ETL
+    class StdoutLoader < Chronicle::ETL::Loader
+      def load(result)
+        puts result.inspect
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -0,0 +1,21 @@
+require 'tty/table'
+
+module Chronicle
+  module ETL
+    class TableLoader < Chronicle::ETL::Loader
+      def initialize(options)
+        super(options)
+      end
+
+      def load(result)
+        @table ||= TTY::Table.new(header: result.keys)
+        values = result.values.map{|x| x.to_s[0..30]}
+        @table << values
+      end
+
+      def finish
+        puts @table.render(:ascii, padding: [0, 1])
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,4 +1,10 @@
-class Chronicle::Etl::Runner
+class Chronicle::ETL::Runner
+  BUILTIN = {
+    extractor: ['stdin', 'json', 'csv', 'file'],
+    transformer: ['null'],
+    loader: ['stdout', 'csv', 'table']
+  }.freeze
+
   def initialize(options)
     @options = options
 
@@ -6,16 +12,18 @@ class Chronicle::Etl::Runner
   end
 
   def run!
-    progress_bar = Chronicle::Etl::Utils::ProgressBarWrapper.new(@extractor.results_count)
-    @loader.start
+    total = @extractor.results_count
+    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
+    count = 0
 
-    @extractor.extract do |result, i|
-      @loader.first_load(result) if i == 0
+    @loader.start
 
-      transformed_data = @transformer.transform(result)
+    @extractor.extract do |data, metadata|
+      transformed_data = @transformer.transform(data)
       @loader.load(transformed_data)
 
       progress_bar.increment
+      count += 1
     end
 
     progress_bar.finish
@@ -25,13 +33,27 @@ class Chronicle::Etl::Runner
   private
 
   def instantiate_etl_classes
-    @extractor = get_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = get_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = get_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
+    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
+    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
+    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
   end
 
-  def get_etl_class(phase, name)
-    klass_name = "Chronicle::Etl::#{phase.to_s.capitalize}s::#{name.capitalize}"
+  def load_etl_class(phase, x)
+    if BUILTIN[phase].include? x
+      klass_name = "Chronicle::ETL::#{x.capitalize}#{phase.to_s.capitalize}"
+    else
+      # TODO: come up with syntax for specifying a particular extractor in a provider library
+      provider, name = x.split(":")
+      provider = x unless provider
+      begin
+        require "chronicle/#{provider}"
+      rescue LoadError => e
+        warn("Error loading #{phase} '#{provider}'".red)
+        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
+        exit(false)
+      end
+      klass_name = "Chronicle::#{provider.capitalize}::#{name&.capitalize}#{phase.capitalize}"
+    end
     Object.const_get(klass_name)
   end
 end