metabase_query_sync 0.1.1

data/lib/metabase_query_sync/metabase_api/pulse.rb

@@ -0,0 +1,97 @@
+class MetabaseQuerySync::MetabaseApi
+  class Pulse < Model
+    KEY = "pulse"
+
+    class Channel < Model
+      attribute :enabled, MetabaseQuerySync::Types::Strict::Bool.default(true)
+      attribute :schedule_type, MetabaseQuerySync::Types::Strict::String
+      attribute :schedule_day, MetabaseQuerySync::Types::Strict::String.optional
+      attribute :schedule_hour, MetabaseQuerySync::Types::Strict::Integer.optional
+      attribute :schedule_frame, MetabaseQuerySync::Types::Strict::String.optional
+      attribute :channel_type, MetabaseQuerySync::Types::Strict::String.enum('email', 'slack')
+      attribute? :recipients, MetabaseQuerySync::Types::Strict::Array do
+        attribute :email, MetabaseQuerySync::Types::Strict::String
+      end
+      attribute? :details do
+        attribute :channel, MetabaseQuerySync::Types::Strict::String
+      end
+
+      def self.build
+        (yield ChannelBuilder.new).()
+      end
+
+      class ChannelBuilder
+        def initialize
+          @args = {}
+          @class = nil
+        end
+
+        # @param hour [Integer] value between 0 and 23
+        def daily(hour)
+          assert_hour hour
+          @args = @args.merge({schedule_type: 'daily', schedule_day: nil, schedule_frame: nil, schedule_hour: hour})
+          self
+        end
+
+        def assert_hour(hour)
+          raise "invalid hour provided (#{hour})" unless (0..23) === hour
+        end
+        def assert_day(day)
+          raise "invalid day provided (#{day})" unless [:sun, :mon, :tue, :wed, :thu, :fri, :sat] === day
+        end
+
+        # @param hour [Integer] value between 0 and 23
+        # @param day [:sun, :mon, :tue, :wed, :thu, :fri, :sat]
+        def weekly(hour, day)
+          assert_hour hour
+          assert_day day
+          @args = @args.merge({
+            schedule_type: 'weekly',
+            schedule_day: day.to_s,
+            schedule_frame: nil,
+            schedule_hour: hour
+          })
+          self
+        end
+
+        def hourly
+          @args = @args.merge({schedule_type: 'hourly', schedule_day: nil, schedule_frame: nil, schedule_hour: nil})
+          self
+        end
+
+        # @param emails [Array<String>]
+        def emails(emails)
+          @args = @args.merge({channel_type: 'email', recipients: emails.map {|e| {email: e}} })
+          self
+        end
+
+        # @param channel [String]
+        def slack(channel)
+          @args = @args.merge({channel_type: 'slack', details: {channel: channel}})
+          self
+        end
+
+        def call()
+          Channel.new(@args)
+        end
+      end
+    end
+
+    class Card < Model
+      attribute :id, MetabaseQuerySync::Types::Strict::Integer
+      attribute :include_csv, MetabaseQuerySync::Types::Strict::Bool.default(false)
+      attribute :include_xls, MetabaseQuerySync::Types::Strict::Bool.default(false)
+    end
+
+    has :id, :archived, :name, :collection_id
+    attribute :cards, MetabaseQuerySync::Types::Strict::Array.of(Pulse::Card)
+    attribute :channels, MetabaseQuerySync::Types::Strict::Array.of(Pulse::Channel)
+    attribute :skip_if_empty, MetabaseQuerySync::Types::Strict::Bool
+
+    # @param put_pulse_request [PutPulseRequest]
+    # @return self
+    def self.from_request(put_pulse_request)
+      new(put_pulse_request.to_h)
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/put_card_request.rb

@@ -0,0 +1,16 @@
+class MetabaseQuerySync::MetabaseApi
+  class PutCardRequest < Model
+    has :id, :archived, :name, :description, :collection_id
+    attribute :display, MetabaseQuerySync::Types::Strict::String.default('table'.freeze)
+    attribute :visualization_settings, MetabaseQuerySync::Types::Strict::Hash.default({}.freeze)
+    attribute :dataset_query, Card::DatasetQuery
+
+    def self.native(sql:, database_id:, **kwargs)
+      new(dataset_query: Card::DatasetQuery.native(sql: sql, database_id: database_id), **kwargs)
+    end
+
+    def self.from_card(card)
+      new(card.to_h)
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/put_collection_request.rb

@@ -0,0 +1,13 @@
+class MetabaseQuerySync::MetabaseApi
+  class PutCollectionRequest < Model
+    has :id, :name, :description, :archived
+    attribute :color, MetabaseQuerySync::Types::Strict::String.default('#509EE3'.freeze)
+    attribute :parent_id, MetabaseQuerySync::Types::Strict::Integer.optional.default(nil)
+
+    # TODO: implement collections
+    # # @param collection [Collection]
+    # def self.from_collection(collection)
+    #   new(collection.to_h)
+    # end
+  end
+end

data/lib/metabase_query_sync/metabase_api/put_pulse_request.rb

@@ -0,0 +1,12 @@
+class MetabaseQuerySync::MetabaseApi
+  class PutPulseRequest < Model
+    has :id, :name, :archived, :collection_id
+    attribute :cards, MetabaseQuerySync::Types::Strict::Array.of(Pulse::Card)
+    attribute :channels, MetabaseQuerySync::Types::Strict::Array.of(Pulse::Channel)
+    attribute :skip_if_empty, MetabaseQuerySync::Types::Strict::Bool
+
+    def self.from_pulse(pulse)
+      new(pulse.to_h)
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/session.rb

@@ -0,0 +1,7 @@
+class MetabaseQuerySync::MetabaseApi
+  # @!method id
+  #   @return [String]
+  class Session < Model
+    attribute :id, MetabaseQuerySync::Types::Strict::String
+  end
+end

data/lib/metabase_query_sync/metabase_api/stub_metabase_api.rb

@@ -0,0 +1,96 @@
+require 'dry-monads'
+
+class MetabaseQuerySync::MetabaseApi
+  class StubMetabaseApi < self
+    include Dry::Monads[:result]
+
+    attr_reader :requests
+
+    def initialize(collections: [], pulses: [], cards: [], databases: [])
+      @collections = collections
+      @pulses = pulses
+      @cards = cards
+      @databases = databases
+      @requests = []
+    end
+
+    def get_collection(id)
+      find_by_id(@collections, id)
+    end
+
+    def get_card(id)
+      find_by_id(@cards, id)
+    end
+
+    def get_databases
+      Success(@databases)
+    end
+
+    def get_pulse(id)
+      find_by_id(@pulses, id)
+    end
+
+    def put_pulse(pulse_request)
+      put_req(Pulse, pulse_request, @pulses) { |pulses| @pulses = pulses }
+    end
+
+    def put_collection(collection_request)
+      @requests << collection_request
+      Success(collection_request)
+    end
+
+    def put_card(card_request)
+      put_req(Card, card_request, @cards) { |cards| @cards = cards }
+    end
+
+    def get_collection_items(collection_id)
+      get_collection(collection_id).bind do
+        Success(@collections.chain(@pulses, @cards).filter do |item|
+          case item
+          when Collection
+            item.parent_id == collection_id
+          when Pulse
+            item.collection_id == collection_id
+          when Card
+            item.collection_id == collection_id
+          else
+            false
+          end
+        end.map do |item|
+          Item.new(id: item.id, name: item.name, description: item.respond_to?(:description) ? item.description : nil, model: case item
+          when Collection
+            Collection::KEY
+          when Pulse
+            Pulse::KEY
+          when Card
+            Card::KEY
+          end)
+        end)
+      end
+    end
+
+    private
+
+    def find_by_id(items, id)
+      item = items.find { |i| i.id == id }
+      item ? Success(item) : Failure(nil)
+    end
+
+    def match_id(id)
+      ->(item) { item.id == id }
+    end
+
+    def put_req(klass, req, collection)
+      @requests << req
+      item = klass.from_request(req)
+      if item.id
+        collection = collection.map { |i| i.id == item.id ? item : i }
+      else
+        item = item.new(id: collection.map { |i| i.id }.compact.max.to_i + 1)
+        collection << item
+      end
+      yield collection
+      Success(item)
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api.rb

@@ -0,0 +1,37 @@
+# Lightweight metabase api interface to be just enough
+# for this project.
+class MetabaseQuerySync::MetabaseApi
+  # collections
+
+  def get_collection(id); throw; end
+  def get_collection_items(collection_id); throw; end
+  # @param collection_request [PutCollectionRequest]
+  def put_collection(collection_request); throw; end
+
+  # cards
+
+  def get_card(id); throw; end
+  # @param [PutCardRequest]
+  def put_card(card_request); throw; end
+  def delete_card(card_id); throw; end
+
+  # pulses
+
+  def get_pulse(id); throw; end
+  # @param [PutPulseRequest]
+  def put_pulse(pulse_request); throw; end
+  def delete_pulse(pulse_id); throw; end
+
+  # database
+
+  def get_databases(); throw; end
+
+  # search
+  def search(q, model: nil); throw; end
+
+  private
+
+  def throw
+    raise 'not implemented'
+  end
+end

data/lib/metabase_query_sync/metabase_credentials.rb

@@ -0,0 +1,18 @@
+class MetabaseQuerySync::MetabaseCredentials
+  attr_reader :host, :user, :pass
+
+  def initialize(host:, user:, pass:)
+    raise "Metabase credentials for host, user, pass must not be empty)" if host == nil || user == nil || pass == nil
+    @host = host
+    @user = user
+    @pass = pass
+  end
+
+  def self.from_env(host: nil, user: nil, pass: nil, env: ENV)
+    self.new(
+      host: host || env['METABASE_QUERY_SYNC_HOST'],
+      user: user || env['METABASE_QUERY_SYNC_USER'],
+      pass: pass || env['METABASE_QUERY_SYNC_PASS'],
+    )
+  end
+end

data/lib/metabase_query_sync/metabase_state.rb

@@ -0,0 +1,82 @@
+require 'dry-struct'
+
+# Holds all of the data/state that has been previously synced to metabase
+module MetabaseQuerySync
+  # @!method collections
+  #   @return [Array<MetabaseApi::Collection>]
+  # @!method cards
+  #   @return [Array<MetabaseApi::Card>]
+  # @!method pulses
+  #   @return [Array<MetabaseApi::Pulse>]
+  # @!method databases
+  #   @return [Array<MetabaseApi::Database>]
+  class MetabaseState < Dry::Struct
+    attribute :collections, Types::Strict::Array.of(MetabaseApi::Collection)
+    attribute :cards, Types::Strict::Array.of(MetabaseApi::Card)
+    attribute :pulses, Types::Strict::Array.of(MetabaseApi::Pulse)
+    attribute :databases, Types::Strict::Array.of(MetabaseApi::Database)
+
+    # @param metabase_api [MetabaseApi]
+    # @param root_collection_id [Integer]
+    # @return [MetabaseState]
+    def self.from_metabase_api(metabase_api, root_collection_id)
+      items = metabase_api.get_collection_items(root_collection_id)
+      if items.failure?
+        raise "No root collection (id: #{root_collection_id}) found"
+      end
+
+      acc = items.value!
+        .filter { |i| i.card? || i.pulse? }
+        .map do |item|
+          if item.card?
+            metabase_api.get_card(item.id).value!
+          elsif item.pulse?
+            metabase_api.get_pulse(item.id).value!
+          else
+            raise 'Unexpected item type.'
+          end
+        end
+        .reduce({cards: [], pulses: []}) do |acc, item|
+          case item
+          when MetabaseApi::Card
+            acc[:cards] << item
+          when MetabaseApi::Pulse
+            acc[:pulses] << item
+          else
+            raise 'Unexpected item type.'
+          end
+          acc
+        end
+
+      new(collections: [], cards: acc[:cards], pulses: acc[:pulses], databases: metabase_api.get_databases.value!)
+    end
+
+    # @return self
+    def with_card(card)
+      new(cards: cards.concat([card]))
+    end
+
+    # @return self
+    def with_pulse(pulse)
+      new(pulses: pulses.concat([pulse]))
+    end
+
+    def empty?
+      collections.empty? && cards.empty? && pulses.empty?
+    end
+
+    # @return [MetabaseApi::Pulse, nil]
+    def pulse_by_name(name)
+      pulses.filter { |p| p.name.downcase == name.downcase }.first
+    end
+
+    # @return [MetabaseApi::Card, nil]
+    def card_by_name(name)
+      cards.filter { |c| c.name.downcase == name.downcase }.first
+    end
+
+    def database_by_name(name)
+      databases.filter { |d| d.name.downcase == name.downcase }.first
+    end
+  end
+end

data/lib/metabase_query_sync/read_ir/from_files.rb

@@ -0,0 +1,20 @@
+require 'yaml'
+
+class MetabaseQuerySync::ReadIR
+  class FromFiles < self
+    def initialize(path)
+      @path = path
+    end
+
+    def call
+      MetabaseQuerySync::IR::Graph.from_items(
+        # @type [String] f
+        Dir[File.join(@path, "**/*.{query,pulse}.yaml")].map do |f|
+          data = YAML.load_file(f)
+          next MetabaseQuerySync::IR::Query.from_h(data) if f.end_with? 'query.yaml'
+          next MetabaseQuerySync::IR::Pulse.from_h(data) if f.end_with? 'pulse.yaml'
+        end
+      )
+    end
+  end
+end

data/lib/metabase_query_sync/read_ir.rb

@@ -0,0 +1,7 @@
+# ReadIR from source and load up instances of an IR root model
+class MetabaseQuerySync::ReadIR
+  # @return [MetabaseQuerySync::IR::Graph]
+  def call()
+    raise 'not implemented.'
+  end
+end

data/lib/metabase_query_sync/sync.rb

@@ -0,0 +1,167 @@
+require 'logger'
+
+module MetabaseQuerySync
+  # @!attribute metabase_api
+  #   @return [MetabaseApi]
+  class Sync
+    def initialize(read_ir, metabase_api, logger = nil)
+      @read_ir = read_ir
+      @metabase_api = metabase_api
+      @logger = logger || Logger.new(IO::NULL)
+    end
+
+    # @param config [Config]
+    def self.from_config(config, logger = nil)
+      new(ReadIR::FromFiles.new(config.path), MetabaseApi::FaradayMetabaseApi.from_metabase_credentials(config.credentials), logger)
+    end
+
+    # @param sync_req [SyncRequest]
+    def call(sync_req)
+      @logger.info "Starting sync with req: #{sync_req.to_h}"
+      graph = @read_ir.()
+      metabase_state = MetabaseState.from_metabase_api(@metabase_api, sync_req.root_collection_id)
+
+      metabase_state = sync_requests(calc_cards_diff(graph, metabase_state, sync_req.root_collection_id), sync_req.dry_run, metabase_state)
+      metabase_state = sync_requests(calc_pulses_diff(graph, metabase_state, sync_req.root_collection_id), sync_req.dry_run, metabase_state)
+
+      @logger.info "Finished sync"
+    end
+
+    private
+
+    def calc_cards_diff(graph, metabase_state, root_collection_id)
+      [].chain(
+        delete_cards(graph, metabase_state),
+        add_cards(graph, metabase_state, root_collection_id)
+      )
+    end
+
+    # pulses need to be synced after the cards since pulses need to make use of card ids
+    def calc_pulses_diff(graph, metabase_state, root_collection_id)
+      [].chain(
+        delete_pulses(graph, metabase_state),
+        add_pulses(graph, metabase_state, root_collection_id)
+      )
+    end
+
+    # @param graph [IR::Graph]
+    # @param metabase_state [MetabaseState]
+    def delete_pulses(graph, metabase_state)
+      metabase_state.pulses
+        .filter { |pulse| graph.pulse_by_name(pulse.name) == nil }
+        .map { |pulse| MetabaseApi::PutPulseRequest.from_pulse(pulse).new(archived: true) }
+    end
+
+    # @param graph [IR::Graph]
+    # @param metabase_state [MetabaseState]
+    def delete_cards(graph, metabase_state)
+      metabase_state.cards
+        .filter { |card| graph.query_by_name(card.name) == nil }
+        .map { |card| MetabaseApi::PutCardRequest.from_card(card).new(archived: true) }
+    end
+
+    # @param graph [IR::Graph]
+    # @param metabase_state [MetabaseState]
+    # @param root_collection_id [Integer]
+    def add_cards(graph, metabase_state, root_collection_id)
+      graph.queries
+        .map do |q|
+          [q, metabase_state.card_by_name(q.name)]
+        end
+        .filter do |(q, card)|
+          next true unless card
+          card.dataset_query.native.query != q.sql ||
+            card.database_id != metabase_state.database_by_name(q.database)&.id ||
+            card.description != q.description
+        end
+        .map do |(q, card)|
+          database_id = metabase_state.database_by_name(q.database)&.id
+          raise "Database (#{q.database}) not found" if database_id == nil
+          MetabaseApi::PutCardRequest.native(
+            id: card&.id,
+            sql: q.sql,
+            database_id: metabase_state.database_by_name(q.database)&.id,
+            name: q.name,
+            description: q.description,
+            collection_id: root_collection_id,
+          )
+        end
+    end
+
+    # @param graph [IR::Graph]
+    # @param metabase_state [MetabaseState]
+    # # @param root_collection_id [Integer]
+    def add_pulses(graph, metabase_state, root_collection_id)
+      graph.pulses
+        .map do |pulse|
+          api_pulse = metabase_state.pulse_by_name(pulse.name)
+          pulse_cards = graph
+            .queries_by_pulse(pulse.name)
+            .flat_map do |query|
+              card = metabase_state.card_by_name(query.name)
+              card ? [card] : []
+            end
+            .map { |card| MetabaseApi::Pulse::Card.new(id: card.id) }
+          pulse_channels = pulse.alerts.map do |alert|
+            MetabaseApi::Pulse::Channel.build do |c|
+              case alert.type
+              when 'email'
+                c.emails alert.email.emails
+              when 'slack'
+                c.slack alert.slack.channel
+              end
+
+              case alert.schedule.type
+              when 'hourly'
+                c.hourly
+              when 'daily'
+                c.daily(alert.hour)
+              when 'weekly'
+                c.weekly(alert.hour, alert.day)
+              end
+            end
+          end
+          [pulse, api_pulse, pulse_cards, pulse_channels]
+        end
+        .filter do |(pulse, api_pulse, pulse_cards, pulse_channels)|
+          next true unless api_pulse
+          api_pulse.cards != pulse_cards || api_pulse.channels != pulse_channels
+        end
+        .map do |(pulse, api_pulse, pulse_cards, pulse_channels)|
+          MetabaseApi::PutPulseRequest.new(
+            id: api_pulse&.id,
+            name: pulse.name,
+            cards: pulse_cards,
+            channels: pulse_channels,
+            collection_id: root_collection_id,
+            skip_if_empty: true,
+          )
+        end
+    end
+
+    # sync requests up to metabase
+    # @param requests [Enumerator]
+    # @param metabase_state [MetabaseState]
+    def sync_requests(requests, dry_run, metabase_state)
+      requests.reduce(metabase_state) do |metabase_state, req|
+        case req
+        when MetabaseApi::PutPulseRequest
+          @logger.info "PutPulseRequest #{req.to_h}"
+          next metabase_state if dry_run
+          @metabase_api.put_pulse(req).fmap do |pulse|
+            metabase_state.with_pulse(pulse)
+          end.value!
+        when MetabaseApi::PutCardRequest
+          @logger.info "PutCardRequest #{req.to_h}"
+          next metabase_state if dry_run
+          @metabase_api.put_card(req).fmap do |card|
+            metabase_state.with_card(card)
+          end.value!
+        else
+          @logger.error "Unhandled Request Type: #{req.class}"
+          metabase_state
+        end
+      end
+    end
+  end
+end

data/lib/metabase_query_sync/sync_request.rb

@@ -0,0 +1,12 @@
+require 'dry-struct'
+
+module MetabaseQuerySync
+  # @!method root_collection_id
+  #   @return [Integer]
+  # @!method dry_run
+  #   @return [Boolean]
+  class SyncRequest < Dry::Struct
+    attribute :root_collection_id, Types::Strict::Integer
+    attribute :dry_run, Types::Strict::Bool.default(false)
+  end
+end

data/lib/metabase_query_sync/types.rb

@@ -0,0 +1,3 @@
+module MetabaseQuerySync::Types
+  include Dry.Types
+end

data/lib/metabase_query_sync/version.rb

@@ -0,0 +1,3 @@
+module MetabaseQuerySync
+  VERSION = '0.1.1'
+end

data/lib/metabase_query_sync.rb

@@ -0,0 +1,11 @@
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("ir" => "IR")
+loader.inflector.inflect("read_ir" => "ReadIR")
+loader.inflector.inflect("cli" => "CLI")
+loader.setup
+
+module MetabaseQuerySync
+
+end