importeur 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a5b8cd8b68e60849d7f6fc4921b41d4ff39187a
+  data.tar.gz: 9a23992b2d88fdad38518abc43057d3d585ea11c
+SHA512:
+  metadata.gz: e1e552cb90dc22addb41d5fdaf14cb4774ebdb13952849a46be3f5d20d3d89741078a22498133039f9411e9d3866882b87b1989afedef3291e9c38c7431191ac
+  data.tar.gz: 90056d4a61059cffe16bcd2cf2c6198a8485045568e5cfad40f009cdaff358aedc91865325e95343396c5b6b7c55682b86ca60f9736e277b38e8c7e198ed9138

data/.circleci/config.yml

@@ -0,0 +1,70 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/ruby:2.4.1-node-browsers
+        environment:
+          DATABASE_URL: 'postgres://importeur@localhost/importeur'
+          CAKE_DATA_BUCKET: 'ad2games-cake-data-test-fixtures'
+          AWS_REGION: 'eu-west-1'
+          AWS_ACCESS_KEY_ID: 'aws-access-key-id'
+          AWS_SECRET_ACCESS_KEY: 'aws-secret-access-key' 
+      
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      - image: circleci/postgres:9.6
+        environment:
+          POSTGRES_USER: importeur
+          POSTGRES_DB: importeur
+          POSTGRES_PASSWORD: ''
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-gems
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-gems
+        
+      # Database setup
+      - run: bundle exec rake create_test_db
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+            
+            bundle exec rspec --format progress \
+                            --format RspecJunitFormatter \
+                            --out /tmp/test-results/rspec.xml \
+                            --format progress \
+                            ${TEST_FILES}
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.env

@@ -0,0 +1,8 @@
+APPNEXUS_USERNAME: 'appnexus-username'
+APPNEXUS_PASSWORD: 'appnexus-password'
+CAKE_DATA_BUCKET: 'ad2games-cake-data-test-fixtures'
+AWS_REGION: 'eu-west-1'
+AWS_ACCESS_KEY_ID: 'aws-access-key-id'
+AWS_SECRET_ACCESS_KEY: 'aws-secret-access-key'
+ROCKETFUEL_API_URI: 'https://api-sandbox.rocketfuel.com/2016/'
+ROCKETFUEL_API_AUTH_TOKEN: 'rocketfuel-api-auth-token'

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+.env.test
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.15.4

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2017-10-26)
+
+- Initial release

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in importeur.gemspec
+gemspec
+
+group :development, :test do
+  gem 'rspec_junit_formatter', '0.2.2'
+  gem 'rubocop-ci', github: 'ad2games/rubocop-ci'
+end

data/README.md

@@ -0,0 +1,85 @@
+# Importeur
+
+A universal data import tool (a.k.a. ETL) 🙌
+
+For now it's a single import job, made more composable and put into its own
+place to be used in other projects. The vision is to add more things, so it can
+handle more data sources and targets, eventually.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'importeur'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install importeur
+
+## Usage
+
+The center of it all is the `ETL` class. By inistantiating it with different
+extractors, transformers and loaders, it can be whatever you want it to be:
+
+```ruby
+Importeur::ETL.new(
+  extractor: my_extractor,
+  transformer: my_transformer,
+  loader: my_loader
+)
+```
+
+All three only need to have a `call` method and, hence, can be simple Procs.
+The extractor should return something enumerable, which is then iterated over
+and each item passed into the transformer. The loader receives the enumerable
+result of that.
+
+So far a more or less generic `Extractor` exists.
+
+```ruby
+Importeur::Extractor.new(
+  source,
+  cursor,
+  cursor_key
+)
+```
+
+It takes as arguments a data source that needs to implement
+
+* An `items` method, returning anything enumerable, as well as a
+  `dataset_unique_id`, returning a unique ID for the current version of the
+  dataset
+* A cursor, that needs to implement `read(key)` and `write(key, value)` (e.g.
+  `Rails.cache`) it makes the extractor return `nil` if there is no new data.
+* A key (string), to be passed into the cursor.
+
+An `ActiveRecordPostgresLoader` exists, that has a few very specific
+dependencies. As the name suggests, it imports data into a Postgres database
+using `ActiveRecord`. Additionally, the model used, needs to use
+`acts_as_paranoid`.
+
+Example use-cases can be found in the `spec/integration` directory.
+
+## Development
+
+In order to be able to run the tests, a Postgres database is needed. Having 
+a PostgreSQL user with CREATEDB permission, create `.env.test` file with your 
+DATABASE_URL configuration, for example:
+
+`DATABASE_URL: 'postgres://user:password@localhost:5432/importeur_test'`
+
+Then run `bundle exec rake create_test_db` to create the database and
+`bundle exec rspec` to run tests.
+
+We run `rake rubocop` to make sure, everything looks good.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/ad2games/importeur.

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop-ci'
+
+RSpec::Core::RakeTask.new(:spec)
+
+load 'lib/task/db.rake'
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "importeur"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/importeur.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'importeur/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'importeur'
+  spec.version       = Importeur::VERSION
+  spec.authors       = ['Helge Rausch']
+  spec.email         = ['helge@rausch.io']
+
+  spec.summary       = 'Universal data importer'
+  spec.description   = 'Universal data importer'
+  spec.homepage      = 'https://ad2games.com/'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+      'public gem pushes.'
+  end
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'activerecord', '<5'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'bucket_cake'
+  spec.add_development_dependency 'pg'
+  spec.add_development_dependency 'database_cleaner'
+  spec.add_development_dependency 'appnexusapi'
+  spec.add_development_dependency 'rocketfuel_api'
+  spec.add_development_dependency 'dotenv'
+end

data/lib/importeur/active_record_postgres_loader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Importeur
+  class ActiveRecordPostgresLoader
+    BATCH_SIZE = 500
+
+    def initialize(model, primary_key)
+      @model = model
+      @primary_key = primary_key
+    end
+
+    def call(data)
+      imported_ids = []
+      seen = Set.new
+      data.each_slice(BATCH_SIZE) do |batch|
+        batch_ids = batch.map { |attrs| attrs[primary_key] }
+        imported_ids.concat(batch_ids)
+        records = records_for_batch(batch_ids)
+        store_batch(batch, records, seen)
+      end
+      delete_old_records(imported_ids)
+    end
+
+    private
+
+    attr_reader :model, :primary_key
+
+    def store_batch(batch, records, seen)
+      batch.each do |attrs|
+        next unless seen.add?(attrs[primary_key])
+        store_record(attrs, records)
+      end
+    end
+
+    def store_record(attrs, records)
+      record = records.fetch(attrs[primary_key], model.new)
+      record.assign_attributes(attrs)
+      record.deleted_at = nil
+      return unless record.changed?
+      record.imported_at = Time.now
+      record.save!
+    end
+
+    def records_for_batch(batch_ids)
+      model
+        .with_deleted
+        .where(primary_key => batch_ids)
+        .index_by(&primary_key)
+    end
+
+    def delete_old_records(imported_ids)
+      # Basically `self.class.model.where.not(primary_key => imported_ids)`, but
+      # more efficient in this case.
+      model
+        .joins(<<-SQL)
+          LEFT JOIN (SELECT unnest(ARRAY[#{imported_ids.join(',')}]::int[]) AS primary_key) AS imported
+                 ON imported.primary_key = #{model.table_name}.#{primary_key}
+        SQL
+        .where('imported.primary_key' => nil)
+        .delete_all
+    end
+  end
+end

data/lib/importeur/data_sources/appnexus.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Importeur
+  module DataSources
+    class Appnexus
+      def initialize(appnexus_service)
+        @appnexus_service = appnexus_service
+      end
+
+      def dataset_unique_id
+        md5 = Digest::MD5.new
+        items.each { |item| md5.update(item.to_s) }
+        md5.hexdigest
+      end
+
+      def items
+        @items ||= appnexus_service.get_all
+      end
+
+      private
+
+      attr_reader :appnexus_service
+    end
+  end
+end

data/lib/importeur/data_sources/combined.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Importeur
+  module DataSources
+    class Combined
+      def initialize(*entity_classes)
+        @entity_classes = entity_classes
+      end
+
+      def dataset_unique_id
+        data_sources.map(&:dataset_unique_id).join
+      end
+
+      def items
+        Enumerator.new do |y|
+          data_sources.each do |data_source|
+            data_source.items.each do |entity|
+              y << entity
+            end
+          end
+        end
+      end
+
+      private
+
+      attr_reader :entity_classes
+
+      def data_sources
+        @data_sources ||= entity_classes.map(&:new)
+      end
+    end
+  end
+end

data/lib/importeur/data_sources/rocketfuel.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Importeur
+  module DataSources
+    class Rocketfuel
+      def initialize(rocketfuel_service, params = {})
+        @rocketfuel_service = rocketfuel_service
+        @params = params
+      end
+
+      def dataset_unique_id
+        md5 = Digest::MD5.new
+        items.each { |item| md5.update(item.to_s) }
+        md5.hexdigest
+      end
+
+      def items
+        @items ||= rocketfuel_service.get_all(params)
+      end
+
+      private
+
+      attr_reader :rocketfuel_service, :params
+    end
+  end
+end

data/lib/importeur/etl.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Importeur
+  class ETL
+    def initialize(extractor:, transformer:, loader:)
+      @extractor = extractor
+      @transformer = transformer
+      @loader = loader
+    end
+
+    def call
+      extracted = extractor.call
+      return if extracted.nil?
+      transformed = extracted.lazy.flat_map do |entity|
+        transformer.call(entity)
+      end
+      loader.call(transformed.lazy)
+    end
+
+    private
+
+    attr_reader :extractor, :transformer, :loader
+  end
+end

data/lib/importeur/extractor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Importeur
+  class Extractor
+    def initialize(data_source, cursor, cursor_key)
+      @data_source = data_source
+      @cursor = cursor
+      @cursor_key = cursor_key
+    end
+
+    def call
+      return unless has_new_data?
+      Enumerator.new do |y|
+        feed.each do |item|
+          y << item
+        end
+        store_new_dataset_id
+        clear_current_dataset_id
+      end
+    end
+
+    private
+
+    attr_reader :current_dataset_id, :cursor, :cursor_key, :data_source
+
+    def has_new_data?
+      last_known_id = cursor.read(cursor_key)
+      current_dataset_id != last_known_id
+    end
+
+    def store_new_dataset_id
+      cursor.write(cursor_key, current_dataset_id)
+    end
+
+    def clear_current_dataset_id
+      @current_dataset_id = nil
+    end
+
+    def current_dataset_id
+      @current_dataset_id ||= dataset_id
+    end
+
+    def dataset_id
+      data_source.dataset_unique_id
+    end
+
+    def feed
+      @feed ||= data_source.items
+    end
+  end
+end

data/lib/importeur/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Importeur
+  VERSION = '0.1.0'
+end

data/lib/importeur.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'importeur/version'
+require 'importeur/etl'
+require 'importeur/extractor'
+require 'importeur/active_record_postgres_loader'
+require 'importeur/data_sources/combined'
+require 'importeur/data_sources/appnexus'
+require 'importeur/data_sources/rocketfuel'
+
+module Importeur
+  def self.root
+    Pathname.new(File.dirname(__dir__))
+  end
+end

data/lib/task/db.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'dotenv'
+Dotenv.load('.env.test')
+
+require 'active_record'
+
+desc 'Create a database for testing'
+task :create_test_db do
+  database_configuration =
+    ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(
+      ENV.fetch('DATABASE_URL')
+    ).to_hash
+
+  ActiveRecord::Tasks::DatabaseTasks.drop(database_configuration)
+  ActiveRecord::Tasks::DatabaseTasks.create(database_configuration)
+
+  ActiveRecord::Base.establish_connection(ENV.fetch('DATABASE_URL'))
+  ActiveRecord::Migration.verbose = false
+  ActiveRecord::Schema.define(version: 1) do
+    drop_table :affiliates if table_exists?(:affiliates)
+    create_table :affiliates do |t|
+      t.string :name
+      t.datetime :deleted_at
+      t.datetime :imported_at
+    end
+  end
+end

metadata

@@ -0,0 +1,247 @@
+--- !ruby/object:Gem::Specification
+name: importeur
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Helge Rausch
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-10-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bucket_cake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: appnexusapi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rocketfuel_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Universal data importer
+email:
+- helge@rausch.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".env"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- importeur.gemspec
+- lib/importeur.rb
+- lib/importeur/active_record_postgres_loader.rb
+- lib/importeur/data_sources/appnexus.rb
+- lib/importeur/data_sources/combined.rb
+- lib/importeur/data_sources/rocketfuel.rb
+- lib/importeur/etl.rb
+- lib/importeur/extractor.rb
+- lib/importeur/version.rb
+- lib/task/db.rake
+homepage: https://ad2games.com/
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Universal data importer
+test_files: []