safra 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 23e8b482c9f993d696c464b3b1ee7853644f297903c23841b3e92c032413da1d
+  data.tar.gz: 305f791964ffd4115c45af7d4f6757eab9eadab5513fa6bf6bc6aec7d00785a2
+SHA512:
+  metadata.gz: '09c1f4a12ea0308f25330c83ac8b06359ab90740c451b7ead7cf5f133dbefc254da0d677badc60f04a66c7ceeafc514b84a3544a890a1642c74da4fd35a2b92e'
+  data.tar.gz: cb0a781efe89bf7cdd84ecd74311e151ae4b858233325601135908d2d93f693e06cf707c1d0043d098e42ed6bfe3c11c0e4be2402a18b216491947b537db648d

data/lib/generators/safra/install_generator.rb

@@ -0,0 +1,22 @@
+require "rails/generators/active_record"
+
+module Safra
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_model_file
+        template "request.rb.tt", "app/models/request.rb"
+      end
+
+      def copy_migration
+        migration_template "requests_migration.rb", "db/migrate/create_requests.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/safra/pipeline_generator.rb

@@ -0,0 +1,20 @@
+
+module Extractor
+  module Generators
+    class PipelineGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates', __dir__)
+
+      def copy_cleaned_file
+        template "pipeline/cleaned.sql.tt", "app/sql/#{file_name}/#{file_name}_cleaned.sql"
+      end
+
+      def copy_deduplicated_file
+        template "pipeline/deduplicated.sql.tt", "app/sql/#{file_name}/#{file_name}_cleaned_deduplicated.sql"
+      end
+
+      def copy_model_file
+        template "pipeline/model.sql.tt", "app/sql/#{file_name}/#{file_name}_model.sql"
+      end
+    end
+  end
+end

data/lib/generators/safra/requests_generator.rb

@@ -0,0 +1,22 @@
+require "rails/generators/active_record"
+
+module Extractor
+  module Generators
+    class RequestsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_model_file
+        template "request.rb.tt", "app/models/request.rb"
+      end
+
+      def copy_migration
+        migration_template "requests_migration.rb", "db/migrate/create_requests.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/safra/tap_generator.rb

@@ -0,0 +1,16 @@
+
+module Safra
+  module Generators
+    class TapGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates', __dir__)
+
+      def copy_tap_file
+        template "tap.rb.tt", "app/extractors/#{file_name}_tap.rb"
+      end
+
+      def copy_sql_file
+        template "dbt.sql.tt", "app/sql/#{file_name}.sql"
+      end
+    end
+  end
+end

data/lib/generators/safra/templates/dbt.sql.tt

@@ -0,0 +1,5 @@
+
+select
+    response_options -> 'response_body' body
+from <%%= source 'requests' %>
+where extractor_class = '<%= "#{class_name}Tap" %>'

data/lib/generators/safra/templates/pipeline/cleaned.sql.tt

@@ -0,0 +1,8 @@
+SELECT
+  jsonb_path_query(response_options, '$.response_body.data[*]') as body,
+  jsonb_path_query(response_options, '$.response_body.data[*].id') as unique_by,
+  id as request_id,
+  created_at,
+  account_id
+FROM <%%= source 'requests' %>
+WHERE extractor_class = '<%= "#{class_name}Tap" %>'

data/lib/generators/safra/templates/pipeline/deduplicated.sql.tt

@@ -0,0 +1,9 @@
+<%%= build_as :incremental %>
+SELECT
+  DISTINCT ON(unique_by)
+  *
+FROM <%%= ref('<%= "#{file_name}_cleaned" %>') %>
+<%% if is_incremental %>
+  WHERE created_at > (SELECT coalesce(max(created_at), '-infinity'::DATE) FROM <%%= this %>)
+<%% end %>
+ORDER BY unique_by, created_at DESC

data/lib/generators/safra/templates/pipeline/model.sql.tt

@@ -0,0 +1,7 @@
+SELECT
+  unique_by,
+  body,
+  created_at,
+  account_id,
+  request_id
+FROM <%%= ref('<%= "#{file_name}_cleaned_deduplicated" %>') %>

data/lib/generators/safra/templates/request.rb.tt

@@ -0,0 +1,19 @@
+class Request < ApplicationRecord
+  class << self
+    Dir.glob('app/extractors/**/*').map {|t| t.scan /extractors\/(.*_tap)\.rb/}.flatten.each do |name|
+      define_method(name) {where extractor_class: name.classify}
+    end
+
+    def create_from_response typhoeus_response
+      res = Extractor::ResponseWithJson.from_response typhoeus_response
+      create!({
+        extractor_class: self.class,
+        base_url: res.request.base_url,
+        request_options: res.request.options,
+        request_original_options: res.request.original_options,
+        response_options: res.parsed_options,
+        request_cache_key: res.request.cache_key
+      })
+    end
+  end
+end

data/lib/generators/safra/templates/requests_migration.rb.tt

@@ -0,0 +1,16 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :requests do |t|
+      t.string :extractor_class
+      t.string :account_id
+      t.string :base_url
+      t.jsonb :request_options
+      t.jsonb :request_original_options
+      t.jsonb :response_options
+      t.string :request_cache_key
+      t.jsonb :aux
+
+      t.timestamps
+    end
+  end
+end

data/lib/generators/safra/templates/tap.rb.tt

@@ -0,0 +1,20 @@
+class <%= class_name %>Tap < Safra::Tap
+
+  # Value counts up from 1 by default
+  # Customize defining first_value and next_value methods
+  def request_for value
+    Typhoeus.get "https://jsonplaceholder.typicode.com/posts/#{value}"
+  end
+
+  # Use response.json for a parsed body (nil if invalid json)
+  # Return anything other than nil or false if reached the end
+  def reached_end? response
+    response.body == "{}"
+  end
+
+  # Return nil, false or explode this method on invalid response
+  def validate response
+    Integer(response.json['id'])
+  end
+
+end

data/lib/generators/safra/templates/update_requests_migration.rb.tt

@@ -0,0 +1,5 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :requests, :aux, :jsonb
+  end
+end

data/lib/generators/safra/update_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Extractor
+  module Generators
+    class UpdateGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "update_requests_migration.rb", "db/migrate/update_requests.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/safra/response_with_json.rb

@@ -0,0 +1,20 @@
+module Safra
+  class ResponseWithJson < Typhoeus::Response
+    def self.from_response response
+      res = new(response.options)
+      response.request.response = res
+      res.request = response.request
+      res
+    end
+
+    def json
+      JSON.parse(body) rescue nil
+    end
+
+    def parsed_options
+      a = options.dup
+      a[:response_body] = JSON.parse(a[:response_body]) rescue a[:response_body]
+      a
+    end
+  end
+end

data/lib/safra/tap.rb

@@ -0,0 +1,151 @@
+module Safra
+  class Tap
+
+    MAX_RETRIES = 4
+    SUPPORTED_ON_MAX_RETRIES_VALUES = [:fail, :save_to_errors, :skip_silently]
+    ON_MAX_RETRIES = :fail
+    MAX_CONCURRENCY = 200
+
+    def initialize parameter=nil, auth:{}
+      check_on_max_retries
+      @auth = auth.with_indifferent_access
+      if @auth.present? and @auth[:account_id].blank?
+        puts "WARNING: No account_id provided in 'auth:' parameter for #{self.class}"
+      end
+      @parameter = parameter
+
+      @request_for_method_name =
+        self.class.instance_methods(false)
+        .find {|m| m.to_s.start_with?('request_for_')} || :request_for
+
+      @request_for_batch_size = [
+        @request_for_method_name
+        .to_s.gsub('request_for_','').to_i,
+        1].max
+
+      if @parameter.is_a? Array
+        @current_value = @parameter.first @request_for_batch_size
+      else
+        @current_value = first_value || 1
+      end
+      @start_time = Time.now.to_i
+    end
+
+    def next_value
+      if @current_value.is_a? Array
+        @parameter -= @current_value
+        @parameter.first @request_for_batch_size
+      else
+        @current_value + 1
+      end
+    end
+
+    def reached_end? res
+      @parameter.empty?
+    end
+
+    def first_value
+      nil
+    end
+
+    def perform
+      @retries_count = 0
+      while @current_value.present? do
+        original_response = send(@request_for_method_name, @request_for_batch_size == 1 ? (@current_value.first rescue @current_value) : @current_value)
+        raise "Function request_for() should return a Typhoeus::Response, but returned #{original_response.class}" if original_response.class != Typhoeus::Response
+        res = ResponseWithJson.from_response original_response
+        response_valid = (validate(res) rescue nil)
+        if response_valid
+          @last_response = res
+          Request.insert! build_request_model(res)
+          @retries_count = 0
+          if (reached_end?(res) rescue false)
+            @current_value = nil
+          else
+            @current_value = next_value
+          end
+        else
+          if (reached_end?(res) rescue false)
+            @current_value = nil
+          elsif @retries_count < self.class::MAX_RETRIES
+            @retries_count += 1
+            puts "sleep #{(2**@retries_count)} then will retry #{'again' if @retries_count > 1}"
+            sleep (2**@retries_count)
+            redo
+          else
+            case self.class::ON_MAX_RETRIES
+            when :fail
+              raise "Maximum number of retries reached (#{self.class::MAX_RETRIES})"
+            when :save_to_errors
+              Request.insert! build_request_model_for_error(res)
+            when :skip_silently
+              puts "skiped on value #{@current_value}"
+            end
+            @current_value = next_value
+            @retries_count = 0
+          end
+        end
+
+      end
+      puts "Completed run of #{self.class}"
+    end
+
+    def parallel_perform
+      @hydra = Typhoeus::Hydra.new(max_concurrency: self.class::MAX_CONCURRENCY)
+      @parameter.each_slice(@request_for_batch_size) do |batch|
+        request = send(@request_for_method_name, (@request_for_batch_size == 1 ? (batch.first rescue batch) : batch))
+        raise "Function request_for() should return a Typhoeus::Request, but returned #{request.class}" if request.class != Typhoeus::Request
+        request.on_complete do |response|
+          res = ResponseWithJson.from_response response
+          response_valid = (validate(res) rescue nil)
+          if response_valid
+            Request.insert! build_request_model(res)
+          else
+            Request.insert! build_request_model_for_error(res)
+          end
+        end
+        @hydra.queue request
+      end
+      @hydra.run
+    end
+
+    private
+    def build_request_model typhoeus_response
+      request_model = {
+        extractor_class: self.class,
+        account_id: @auth[:account_id],
+        base_url: typhoeus_response.request.base_url,
+        request_options: typhoeus_response.request.options,
+        request_original_options: typhoeus_response.request.original_options,
+        response_options: typhoeus_response.parsed_options,
+        request_cache_key: typhoeus_response.request.cache_key
+      }
+      if Request.column_names.include?('aux')
+        request_model[:aux] = { value: @current_value, retries: @retries_count, run_id: "#{object_id}-#{@start_time}"}
+      end
+      request_model
+    end
+
+    def build_request_model_for_error typhoeus_response
+      request_model = {
+        extractor_class: "#{self.class}_errors",
+        account_id: @auth[:account_id],
+        base_url: typhoeus_response.request.base_url,
+        request_options: typhoeus_response.request.options,
+        request_original_options: typhoeus_response.request.original_options,
+        response_options: typhoeus_response.parsed_options,
+        request_cache_key: typhoeus_response.request.cache_key,
+      }
+      if Request.column_names.include?('aux')
+        request_model[:aux] = { value: @current_value, retries: @retries_count, run_id: "#{object_id}-#{@start_time}"}
+      end
+      request_model
+    end
+
+    def check_on_max_retries
+      unless SUPPORTED_ON_MAX_RETRIES_VALUES.include? self.class::ON_MAX_RETRIES
+        raise "Unsuported ON_MAX_RETRIES value #{self.class::ON_MAX_RETRIES} supported values are #{SUPPORTED_ON_MAX_RETRIES_VALUES}"
+      end
+    end
+  end
+end

data/lib/safra.rb

@@ -0,0 +1,8 @@
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.ignore("#{__dir__}/generators")
+loader.setup
+
+module Safra
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: safra
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Felipe Mesquita
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+email: felipemesquita@hey.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/generators/safra/install_generator.rb
+- lib/generators/safra/pipeline_generator.rb
+- lib/generators/safra/requests_generator.rb
+- lib/generators/safra/tap_generator.rb
+- lib/generators/safra/templates/dbt.sql.tt
+- lib/generators/safra/templates/pipeline/cleaned.sql.tt
+- lib/generators/safra/templates/pipeline/deduplicated.sql.tt
+- lib/generators/safra/templates/pipeline/model.sql.tt
+- lib/generators/safra/templates/request.rb.tt
+- lib/generators/safra/templates/requests_migration.rb.tt
+- lib/generators/safra/templates/tap.rb.tt
+- lib/generators/safra/templates/update_requests_migration.rb.tt
+- lib/generators/safra/update_generator.rb
+- lib/safra.rb
+- lib/safra/response_with_json.rb
+- lib/safra/tap.rb
+homepage: https://github.com/felipedmesquita/extractor
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Extract data from APIs with mininal configuration
+test_files: []