land 0.1.3

data/app/lookups/land/path.rb

@@ -0,0 +1,10 @@
+module Land
+  class Path < ApplicationRecord
+    include TableName
+
+    lookup_by :path, cache: 50, find_or_create: true
+
+    has_many :page_views
+    has_many :referers
+  end
+end

data/app/lookups/land/placement.rb

@@ -0,0 +1,9 @@
+module Land
+  class Placement < ApplicationRecord
+    include TableName
+
+    lookup_by :placement, cache: 50, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/platform.rb

@@ -0,0 +1,10 @@
+module Land
+  class Platform < ApplicationRecord
+    include TableName
+
+    lookup_by :platform, cache: 50, find_or_create: true
+
+    has_many :attributions
+    has_many :user_agents
+  end
+end

data/app/lookups/land/position.rb

@@ -0,0 +1,9 @@
+module Land
+  class Position < ApplicationRecord
+    include TableName
+
+    lookup_by :position, cache: 50, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/query_string.rb

@@ -0,0 +1,10 @@
+module Land
+  class QueryString < ApplicationRecord
+    include TableName
+
+    lookup_by :query_string, cache: 50, find_or_create: true, allow_blank: true
+
+    has_many :page_views
+    has_many :referers
+  end
+end

data/app/lookups/land/search_term.rb

@@ -0,0 +1,9 @@
+module Land
+  class SearchTerm < ApplicationRecord
+    include TableName
+
+    lookup_by :search_term, cache: 50, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/source.rb

@@ -0,0 +1,9 @@
+module Land
+  class Source < ApplicationRecord
+    include TableName
+
+    lookup_by :source, cache: 250, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/subsource.rb

@@ -0,0 +1,9 @@
+module Land
+  class Subsource < ApplicationRecord
+    include TableName
+
+    lookup_by :subsource, cache: 1000, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/target.rb

@@ -0,0 +1,9 @@
+module Land
+  class Target < ApplicationRecord
+    include TableName
+
+    lookup_by :target, cache: 50, find_or_create: true
+
+    has_many :attributions
+  end
+end

data/app/lookups/land/user_agent_type.rb

@@ -0,0 +1,9 @@
+module Land
+  class UserAgentType < ApplicationRecord
+    include TableName
+
+    lookup_by :user_agent_type, cache: true
+
+    has_many :user_agents
+  end
+end

data/app/models/concerns/land/table_name.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Land
+  module TableName
+    extend ActiveSupport::Concern
+
+    included do
+      self.table_name = "#{Land.config.schema}.#{model_name.element.pluralize}"
+    end
+  end
+end

data/app/models/land/application_record.rb

@@ -0,0 +1,5 @@
+module Land
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/land/attribution.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Land
+  class Attribution < ApplicationRecord
+    include TableName
+
+    KEYS = %w[
+      ad_type
+      ad_group
+      affiliate
+      app
+      bid_match_type
+      brand
+      campaign
+      content
+      creative
+      device_type
+      experiment
+      keyword
+      match_type
+      medium
+      network
+      placement
+      position
+      search_term
+      source
+      subsource
+      target
+    ]
+
+    self.record_timestamps = false
+
+    KEYS.each do |key|
+      lookup_for key.to_sym, class_name: "Land::#{key.classify}".constantize
+    end
+
+    has_many :visits
+
+    class << self
+      def transform(params)
+        hash = params.slice(*KEYS)
+
+        filter = {}
+
+        hash.each do |k, v|
+          filter[k.foreign_key] = "Land::#{k.classify}".constantize[v]
+        end
+
+        filter
+      end
+
+      def lookup(params)
+        where(transform(params)).first_or_create
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+
+      def digest(params)
+        Digest::SHA2.base64digest transform(params).values.map(&:name).sort.join("\n")
+      end
+    end
+  end
+end

data/app/models/land/cookie.rb

@@ -0,0 +1,14 @@
+module Land
+  class Cookie < ApplicationRecord
+    include TableName
+
+    lookup_by :cookie_id, cache: 100, find: true
+
+    has_many :ownerships
+    has_many :visits
+
+    after_initialize do
+      self.id ||= SecureRandom.uuid
+    end
+  end
+end

data/app/models/land/event.rb

@@ -0,0 +1,10 @@
+module Land
+  class Event < ApplicationRecord
+    include TableName
+    self.record_timestamps = false
+
+    lookup_for :event_type, class_name: EventType, symbolize: true
+
+    belongs_to :pageview, required: false
+  end
+end

data/app/models/land/owner.rb

@@ -0,0 +1,10 @@
+module Land
+  class Owner < ApplicationRecord
+    include TableName
+
+    lookup_by :owner, cache: 50, find_or_create: true
+
+    has_many :ownerships
+    has_many :visits
+  end
+end

data/app/models/land/ownership.rb

@@ -0,0 +1,8 @@
+module Land
+  class Ownership < ApplicationRecord
+    include TableName
+
+    lookup_for :cookie, class_name: Cookie
+    lookup_for :owner,  class_name: Owner
+  end
+end

data/app/models/land/pageview.rb

@@ -0,0 +1,21 @@
+module Land
+  class Pageview < ApplicationRecord
+    include TableName
+    self.record_timestamps = false
+
+    # optional: true is misleading
+    # visit_id is actually required, but its validity is enforced by the db.
+    # Without this Rails loads the visit record when saving the pageview.
+    # This removes the unnecessary query.
+    belongs_to :visit, optional: true
+
+    lookup_for :mime_type,    class_name: MimeType
+    lookup_for :http_method,  class_name: HttpMethod
+    lookup_for :path,         class_name: Path
+    lookup_for :query_string, class_name: QueryString
+
+    after_initialize do
+      self.id ||= SecureRandom.uuid
+    end
+  end
+end

data/app/models/land/referer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Land
+  class Referer < ApplicationRecord
+    include TableName
+
+    lookup_for :domain,       class_name: Domain
+    lookup_for :path,         class_name: Path
+    lookup_for :query_string, class_name: QueryString
+
+    def url
+      uri.to_s
+    end
+
+    def uri
+      URI.join("https://#{domain}", path)
+    end
+  end
+end

data/app/models/land/user_agent.rb

@@ -0,0 +1,16 @@
+module Land
+  class UserAgent < ApplicationRecord
+    include TableName
+
+    self.record_timestamps = false
+
+    lookup_by :user_agent, cache: 50, find_or_create: true
+
+    lookup_for :user_agent_type, class_name: UserAgentType
+    lookup_for :device,          class_name: Device
+    lookup_for :platform,        class_name: Platform
+    lookup_for :browser,         class_name: Browser
+
+    has_many :visits
+  end
+end

data/app/models/land/visit.rb

@@ -0,0 +1,18 @@
+module Land
+  class Visit < ApplicationRecord
+    include TableName
+
+    belongs_to :attribution
+    belongs_to :cookie
+    belongs_to :user_agent
+    belongs_to :referer, optional: true
+
+    lookup_for :owner, class_name: Owner
+
+    has_many :pageviews
+
+    after_initialize do
+      self.id ||= SecureRandom.uuid
+    end
+  end
+end

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "land"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# require "irb"
+# IRB.start(__FILE__)
+
+require "pry"
+Pry.start

data/bin/rails

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENV['RAILS_ENV'] ||= 'development'
+
+ENGINE_ROOT = File.expand_path('..', __dir__)
+ENGINE_PATH = File.expand_path('../lib/land/engine', __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+require 'combustion'
+
+Combustion.initialize! :active_record,
+  database_reset: false,
+  load_schema: false,
+  database_migrate: false
+
+require 'rails/all'
+require 'rails/engine/commands'

data/bin/setup

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+rails db:reset
+
+rspec
+
+# Do any other automated setup that you need to do here

data/config.ru

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "rubygems"
+require "bundler"
+
+Bundler.require :default, :development
+
+Combustion.initialize! :active_record,
+  database_reset: false,
+  load_schema: false,
+  database_migrate: false
+
+run Combustion::Application

data/db/migrate/20200103012916_create_land_schema.rb

@@ -0,0 +1,227 @@
+class CreateLandSchema < ActiveRecord::Migration[5.0]
+  QUERY_PARAMS = %w[
+    ad_type
+    ad_group
+    affiliate
+    app
+    bid_match_type
+    brand
+    campaign
+    content
+    creative
+    device_type
+    experiment
+    keyword
+    match_type
+    medium
+    network
+    placement
+    position
+    search_term
+    source
+    subsource
+    target
+  ]
+
+  def schema
+    Land.config.schema
+  end
+
+  def up
+    execute "CREATE SCHEMA #{schema};"
+
+    with_options schema: schema do |t|
+      # Query params
+      t.create_lookup_tables(*QUERY_PARAMS.map(&:pluralize))
+
+      # User agent
+      t.create_lookup_table  :devices
+      t.create_lookup_tables :user_agent_types, :browsers, :platforms, small: true
+
+      # HTTP
+      t.create_lookup_tables :domains, :paths, :query_strings
+      t.create_lookup_tables :http_methods, :mime_types, small: true
+
+      t.create_lookup_table :event_types, small: true
+    end
+
+    execute %[CREATE EXTENSION "uuid-ossp";]
+
+    enable_extension "uuid-ossp"
+
+    execute %[ALTER EXTENSION "uuid-ossp" SET SCHEMA "public";]
+
+    execute <<~SQL
+      SET search_path TO #{schema},public;
+
+      INSERT INTO user_agent_types (user_agent_type) VALUES ('user'), ('ping'), ('crawl'), ('scrape'), ('scan');
+
+      CREATE TABLE user_agents (
+          user_agent_id      SERIAL      PRIMARY KEY
+
+        , user_agent_type_id SMALLINT    REFERENCES user_agent_types
+
+        , device_id          INTEGER     REFERENCES devices
+        , platform_id        SMALLINT    REFERENCES platforms
+        , browser_id         SMALLINT    REFERENCES browsers
+        , browser_version    TEXT
+
+        , user_agent         TEXT        NOT NULL UNIQUE
+
+        , created_at         TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      );
+
+      CREATE INDEX ON user_agents (device_id);
+      CREATE INDEX ON user_agents (platform_id);
+      CREATE INDEX ON user_agents (browser_id);
+
+      ALTER TABLE ad_types        ALTER COLUMN ad_type_id        SET DATA TYPE SMALLINT;
+      ALTER TABLE bid_match_types ALTER COLUMN bid_match_type_id SET DATA TYPE SMALLINT;
+      ALTER TABLE device_types    ALTER COLUMN device_type_id    SET DATA TYPE SMALLINT;
+      ALTER TABLE match_types     ALTER COLUMN match_type_id     SET DATA TYPE SMALLINT;
+      ALTER TABLE networks        ALTER COLUMN network_id        SET DATA TYPE SMALLINT;
+      ALTER TABLE positions       ALTER COLUMN position_id       SET DATA TYPE SMALLINT;
+
+      CREATE TABLE attributions (
+          attribution_id SERIAL PRIMARY KEY
+
+        , #{QUERY_PARAMS.map { |name| format('%s INTEGER REFERENCES %s', name.foreign_key, name.pluralize) }.join(',') }
+
+        , created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+
+        , UNIQUE (#{QUERY_PARAMS.map(&:foreign_key).join(',')})
+      );
+
+      ALTER TABLE attributions
+          ALTER COLUMN ad_type_id        SET DATA TYPE SMALLINT
+        , ALTER COLUMN bid_match_type_id SET DATA TYPE SMALLINT
+        , ALTER COLUMN device_type_id    SET DATA TYPE SMALLINT
+        , ALTER COLUMN match_type_id     SET DATA TYPE SMALLINT
+        , ALTER COLUMN network_id        SET DATA TYPE SMALLINT
+        , ALTER COLUMN position_id       SET DATA TYPE SMALLINT
+      ;
+
+      #{QUERY_PARAMS.map { |p| "CREATE INDEX ON attributions (#{p.foreign_key});" }.join("\n")}
+
+      CREATE TABLE referers (
+          referer_id      SERIAL  PRIMARY KEY
+
+        , domain_id       INTEGER NOT NULL REFERENCES domains
+        , path_id         INTEGER NOT NULL REFERENCES paths
+        , query_string_id INTEGER NOT NULL REFERENCES query_strings
+
+        , attribution_id  INTEGER NOT NULL REFERENCES attributions
+
+        , UNIQUE (domain_id, path_id, query_string_id, attribution_id)
+      );
+
+      CREATE INDEX ON referers (domain_id);
+      CREATE INDEX ON referers (path_id);
+      CREATE INDEX ON referers (query_string_id);
+
+      CREATE INDEX ON referers (attribution_id);
+
+      CREATE TABLE cookies (
+          cookie_id UUID PRIMARY KEY DEFAULT uuid_generate_v4()
+      );
+
+      CREATE TABLE owners (
+          owner_id SERIAL PRIMARY KEY
+        , owner    TEXT   NOT NULL UNIQUE
+      );
+
+      CREATE TABLE ownerships (
+          ownership_id SERIAL PRIMARY KEY
+
+        , owner_id  INTEGER NOT NULL REFERENCES owners
+        , cookie_id UUID    NOT NULL REFERENCES cookies
+
+        , UNIQUE (owner_id, cookie_id)
+      );
+
+      CREATE TABLE visits (
+          visit_id       UUID                 PRIMARY KEY DEFAULT uuid_generate_v4()
+
+        , cookie_id      UUID        NOT NULL REFERENCES  cookies
+
+        , user_agent_id  INTEGER     NOT NULL REFERENCES  user_agents
+        , attribution_id INTEGER     NOT NULL REFERENCES  attributions
+
+        , referer_id     INTEGER              REFERENCES  referers
+        , owner_id       INTEGER              REFERENCES  owners
+
+        , ip_address     INET        NOT NULL
+
+        , created_at     TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        , updated_at     TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      );
+
+      CREATE INDEX ON visits (cookie_id);
+      CREATE INDEX ON visits (user_agent_id);
+      CREATE INDEX ON visits (attribution_id);
+      CREATE INDEX ON visits (referer_id);
+      CREATE INDEX ON visits (owner_id);
+
+      CREATE TABLE pageviews (
+          pageview_id     UUID     PRIMARY KEY DEFAULT uuid_generate_v4()
+
+        , visit_id        UUID     NOT NULL REFERENCES visits
+        , path_id         INTEGER  NOT NULL REFERENCES paths
+        , query_string_id INTEGER  NOT NULL REFERENCES query_strings
+
+        , mime_type_id    SMALLINT          REFERENCES mime_types
+        , http_method_id  SMALLINT NOT NULL REFERENCES http_methods
+
+        , request_id      UUID
+
+        , click_id        TEXT
+
+        , http_status     INTEGER
+        , response_time   INTEGER
+
+        , created_at      TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      );
+
+      CREATE INDEX ON pageviews (visit_id);
+      CREATE INDEX ON pageviews (path_id);
+      CREATE INDEX ON pageviews (query_string_id);
+      CREATE INDEX ON pageviews (request_id);
+      CREATE INDEX ON pageviews (click_id);
+
+      CREATE TABLE events (
+          event_id      UUID        PRIMARY KEY DEFAULT uuid_generate_v4()
+
+        , event_type_id SMALLINT    NOT NULL REFERENCES event_types
+        , visit_id      UUID        NOT NULL REFERENCES visits
+
+        , meta          JSON
+
+        , created_at    TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      );
+
+      CREATE INDEX ON events (event_type_id);
+      CREATE INDEX ON events (visit_id);
+
+
+      INSERT INTO bid_match_types (bid_match_type) VALUES ('bidded broad'), ('bidded content'), ('bidded exact'), ('bidded phrase');
+      INSERT INTO device_types    (device_type)    VALUES ('computer'), ('mobile'), ('tablet');
+      INSERT INTO match_types     (match_type)     VALUES ('broad'), ('phrase'), ('exact');
+      INSERT INTO networks        (network)        VALUES ('google_search'), ('search_partner'), ('display_network');
+
+      CREATE VIEW response_times_by_path AS SELECT *
+        FROM (
+          SELECT path_id
+               , path
+               , ROUND(AVG(response_time), 3) AS "avg response time (ms)"
+
+          FROM land.pageviews pv
+          JOIN land.paths     p USING (path_id)
+
+          GROUP BY path_id, path
+        )  agg
+
+        ORDER BY agg."avg response time (ms)" DESC
+        ;
+    SQL
+  end
+end