metabase_query_sync 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 020bfbb2b8958b1f6f3ed8067628bfb4942ddc8a751abc787b797c562b1500d8
+  data.tar.gz: ed21dd175d50d34dec40ae0fac3c29b65da668cba7d80c23bcb93d4b527d0a0d
+SHA512:
+  metadata.gz: 6b41d731113fea1a55c85875b762a777d5ce77a27dc6efe9f5088ce22febbbc5a5beccc4cf12260a782a6b4e67fddcc7961941d3be1b747fee5ab747c5ca753a
+  data.tar.gz: 6b646e18d35d39a5da020fa43b3f0d9dbc9e9782fcb440a98371960f8866e659b266ad559f7710acc12de0bc21c9fb7e6176e0fcd09a6cc35c102be5c6ca0634

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# MetabaseQuerySync
+
+MetabaseQuerySync is a tool for automatically syncing metabase queries defined in files to a specific metabase installation.
+
+This enables metabase queries to be maintained with the relevant source code to ease refactoring of models in your application.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'metabase_query_sync'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install metabase-query-sync
+
+## Usage
+
+Build files with `.query.yaml` or `.pulse.yaml` suffix and sync those files up to your metabase instance.
+
+### Files Definitions
+
+```yaml
+# in low-volume-orders.query.yaml
+name: Low Volume Orders
+sql: 'select * from orders'
+database: Local DB # must match name of database in metabase
+pulse: Hourly # must match local pulse name field, throws exception if no pulse is found with that name
+```
+
+```yaml
+# in hourly.pulse.yaml
+name: Hourly
+alerts:
+  - type: email # can be one of slack/email
+    email:
+      emails: ['ragboyjr@icloud.com']
+    # or instead
+    #slack:
+    #  channel: '#test-channel'
+    schedule:
+      type: hourly # can be one of hourly, daily, weekly
+      hour: 5 # number from 0-23, only needed if daily or weekly
+      day: mon # first 3 character of day only needed if weekly
+```
+
+### Running the Sync
+
+Then using the metabase-query-sync cli tool, you can sync those files directly into metabase:
+
+```bash
+Command:
+  metabase-query-sync sync
+
+Usage:
+  metabase-query-sync sync ROOT_COLLECTION_ID PATH
+
+Description:
+  Sync queries/pulses to your metabase root collection
+
+Arguments:
+  ROOT_COLLECTION_ID  	# REQUIRED The root collection id to sync all items under.
+  PATH                	# REQUIRED The path to metabase item files to sync from.
+
+Options:
+  --[no-]dry-run, -d              	# Perform a dry run and do not actually sync to the metabase instance., default: false
+  --host=VALUE, -H VALUE          	# Metabase Host, if not set, will read from env at METABASE_QUERY_SYNC_HOST
+  --user=VALUE, -u VALUE          	# Metabase User, if not set, will read from env at METABASE_QUERY_SYNC_USER
+  --pass=VALUE, -p VALUE          	# Metabase Password, if not set, will read from env at METABASE_QUERY_SYNC_PASS
+  --help, -h                      	# Print this help
+```
+
+## Development
+
+- Install gems with `bundle install`
+- Run tests with `bundle exec rspec`
+
+### TODO
+
+- Support Collections and Syncing with collections
+- Matching IR vs MetabaseApi items should go off of the file name + collection id instead of just the name
+
+## Debugging with Metabase
+
+To setup the local data source for metabase, run `make db`.
+
+Starting the metabase docker container should automatically initialize an empty metabase installation with the main admin user account (ragboyjr@icloud.com / password123).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ragboyjr/metabase-query-sync. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/ragboyjr/metabase-query-sync/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Metabase::Query::Sync project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/ragboyjr/metabase-query-sync/blob/master/CODE_OF_CONDUCT.md).

data/bin/metabase-query-sync

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'dry/cli'
+require 'metabase_query_sync'
+
+Dry::CLI.new(MetabaseQuerySync::CLI).call

data/lib/metabase_query_sync/cli.rb

@@ -0,0 +1,36 @@
+require 'dry/cli'
+
+module MetabaseQuerySync
+  class CLI
+    extend Dry::CLI::Registry
+
+    class Version < Dry::CLI::Command
+      def call
+        puts VERSION
+      end
+    end
+
+    class Sync < Dry::CLI::Command
+      desc 'Sync queries/pulses to your metabase root collection'
+
+      argument :root_collection_id, type: :integer, required: true, desc: 'The root collection id to sync all items under.'
+      argument :path, type: :string, required: true, desc: 'The path to metabase item files to sync from.'
+      option :dry_run, type: :boolean, default: false, aliases: ['-d'], desc: 'Perform a dry run and do not actually sync to the metabase instance.'
+      option :host, type: :string, aliases: ['-H'], desc: 'Metabase Host, if not set, will read from env at METABASE_QUERY_SYNC_HOST'
+      option :user, type: :string, aliases: ['-u'], desc: 'Metabase User, if not set, will read from env at METABASE_QUERY_SYNC_USER'
+      option :pass, type: :string, aliases: ['-p'], desc: 'Metabase Password, if not set, will read from env at METABASE_QUERY_SYNC_PASS'
+
+      def call(root_collection_id:, path:, dry_run: false, host: nil, user: nil, pass: nil)
+        config = MetabaseQuerySync::Config.new(
+          credentials: MetabaseQuerySync::MetabaseCredentials.from_env(host: host, user: user, pass: pass),
+          path: path,
+        )
+        sync = MetabaseQuerySync::Sync.from_config(config, Logger.new(STDOUT))
+        sync.(MetabaseQuerySync::SyncRequest.new(root_collection_id: root_collection_id.to_i, dry_run: dry_run))
+      end
+    end
+
+    register "version", Version, aliases: ['v', '-v', '--version']
+    register "sync", Sync, aliases: ['s']
+  end
+end

data/lib/metabase_query_sync/config.rb

@@ -0,0 +1,12 @@
+module MetabaseQuerySync
+  class Config
+    attr_reader :credentials, :path
+
+    # @param credentials [MetabaseQuerySync::MetabaseCredentials]
+    # @param path [String]
+    def initialize(credentials:, path:)
+      @credentials = credentials
+      @path = path
+    end
+  end
+end

data/lib/metabase_query_sync/ir/collection.rb

@@ -0,0 +1,14 @@
+module MetabaseQuerySync::IR
+  class Collection < Model
+    attribute :name, string
+    attribute :description, string.optional
+    attribute :collection, string.optional
+
+    # TODO: implement collections
+    # validate_with_schema do
+    #   required(:name).filled(:string)
+    #   optional(:description).filled(:string)
+    #   optional(:collection).filled(:string)
+    # end
+  end
+end

data/lib/metabase_query_sync/ir/graph.rb

@@ -0,0 +1,69 @@
+require 'dry-struct'
+require 'set'
+
+module MetabaseQuerySync::IR
+  # @!method collections
+  #   @return [Array<Collection>]
+  # @!method queries
+  #   @return [Array<Card>]
+  # @!method pulses
+  #   @return [Array<Pulse>]
+  class Graph < Dry::Struct
+    attribute :collections, MetabaseQuerySync::Types::Strict::Array.of(Collection)
+    attribute :pulses, MetabaseQuerySync::Types::Strict::Array.of(Pulse)
+    attribute :queries, MetabaseQuerySync::Types::Strict::Array.of(Query)
+
+    def initialize(attributes)
+      super(attributes)
+      assert_traversal
+    end
+
+    # create a struct from a heterogeneous collection of collection, pulse, or queries
+    # @return [Graph]
+    def self.from_items(items)
+      new(items.reduce({collections: [], pulses: [], queries: []}) do |acc, item|
+        case item
+        when Collection
+          acc[:collections] << item
+        when Pulse
+          acc[:pulses] << item
+        when Query
+          acc[:queries] << item
+        else
+          raise "Unexpected type provided from items (#{item.class}), expected instances of only Collection, Pulse, or Query"
+        end
+        acc
+      end)
+    end
+
+    # @return [Query, nil]
+    def query_by_name(name)
+      queries.filter { |query| strcmp(query.name, name) }.first
+    end
+
+    # @return [Pulse, nil]
+    def pulse_by_name(name)
+      pulses.filter { |pulse| strcmp(pulse.name, name) }.first
+    end
+
+    # @return [Array<Query>]
+    def queries_by_pulse(pulse_name)
+      queries.filter { |query| strcmp(query.pulse, pulse_name) }
+    end
+
+    private
+
+    def assert_traversal
+      pulse_names = pulses.map(&:name).map(&:downcase).to_set
+      queries.each do |q|
+        raise "No pulse (#{q.pulse}) found for query (#{q.name})" unless pulse_names === q.pulse.downcase
+      end
+    end
+
+    # @param a [String]
+    # @param b [String]
+    def strcmp(a, b)
+      a.downcase == b.downcase
+    end
+  end
+end

data/lib/metabase_query_sync/ir/model.rb

@@ -0,0 +1,30 @@
+require 'dry-struct'
+require 'dry-schema'
+
+class MetabaseQuerySync::IR::Model < Dry::Struct
+  transform_keys &:to_sym
+
+  def self.string
+    MetabaseQuerySync::Types::Strict::String
+  end
+
+  def self.integer
+    MetabaseQuerySync::Types::Strict::Integer
+  end
+
+  def self.bool
+    MetabaseQuerySync::Types::Strict::Bool
+  end
+
+  def self.array
+    MetabaseQuerySync::Types::Strict::Array
+  end
+
+  def self.validate_with_schema(&schema_def)
+    define_singleton_method :from_h do |h|
+      result = Dry::Schema.JSON(&schema_def).(h)
+      raise "Invalid hash provided: #{result.errors.to_h}" if result.failure?
+      new(h)
+    end
+  end
+end

data/lib/metabase_query_sync/ir/pulse.rb

@@ -0,0 +1,50 @@
+module MetabaseQuerySync::IR
+  class Pulse < Model
+    class Alert < Model
+      TYPES = ['email', 'slack'].freeze
+      class Email < Model
+        attribute :emails, array.of(string)
+      end
+      class Slack < Model
+        attribute :channel, string
+      end
+      class Schedule < Model
+        TYPES = ['hourly', 'daily', 'weekly'].freeze
+        DAYS = [nil, 'sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'].freeze
+        attribute :type, string.enum(*TYPES)
+        attribute :hour, integer.optional.default(nil)
+        attribute :day, string.optional.default(nil).enum(*DAYS)
+      end
+
+      attribute :type, string.enum(*TYPES)
+      attribute :email, Email.optional.default(nil)
+      attribute :slack, Slack.optional.default(nil)
+      attribute :schedule, Schedule
+    end
+
+    attribute :name, string
+    attribute :skip_if_empty, bool.default(true)
+    attribute :alerts, array.of(Alert)
+
+    validate_with_schema do
+      required(:name).filled(:string)
+      optional(:skip_if_empty).value(:bool)
+      required(:alerts).value(:array, min_size?: 1).each do
+        hash do
+          required(:type).value(:filled?, :str?, included_in?: Alert::TYPES)
+          required(:schedule).hash do
+            required(:type).value(:filled?, :str?, included_in?: Alert::Schedule::TYPES)
+            optional(:hour).value(:integer)
+            optional(:day).value(included_in?: Alert::Schedule::DAYS)
+          end
+          optional(:email).hash do
+            required(:emails).value(array[:string], min_size?: 1)
+          end
+          optional(:slack).hash do
+            required(:channel).filled(:string)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metabase_query_sync/ir/query.rb

@@ -0,0 +1,22 @@
+require 'dry-schema'
+
+module MetabaseQuerySync::IR
+  class Query < Model
+    attribute :name, string
+    attribute :description, string.optional.default(nil)
+    attribute :sql, string
+    attribute :database, string
+    attribute :pulse, string
+    attribute :collection, string.optional.default(nil)
+
+    validate_with_schema do
+      required(:name).filled(:string)
+      required(:sql).filled(:string)
+      required(:database).filled(:string)
+      required(:pulse).filled(:string)
+      required(:pulse).filled(:string)
+      optional(:description).filled(:string)
+      optional(:collection).filled(:string)
+    end
+  end
+end

data/lib/metabase_query_sync/ir.rb

@@ -0,0 +1,3 @@
+# Internal Representation
+module MetabaseQuerySync::IR
+end

data/lib/metabase_query_sync/metabase_api/card.rb

@@ -0,0 +1,33 @@
+class MetabaseQuerySync::MetabaseApi
+  class Card < Model
+    KEY = "card"
+
+    class DatasetQuery < Model
+      attribute :type, MetabaseQuerySync::Types::Strict::String.default('native'.freeze)
+      attribute :native do
+        attribute :query, MetabaseQuerySync::Types::Strict::String
+      end
+      attribute :database, MetabaseQuerySync::Types::Strict::Integer
+
+      def self.native(sql:, database_id:)
+        new(type: 'native', native: {query: sql}, database: database_id)
+      end
+    end
+
+    has :id, :archived, :name, :description, :collection_id
+    attribute :database_id, MetabaseQuerySync::Types::Strict::Integer
+    attribute :query_type, MetabaseQuerySync::Types::Strict::String.default('native'.freeze)
+    attribute :display, MetabaseQuerySync::Types::Strict::String.default('table'.freeze)
+    attribute :visualization_settings, MetabaseQuerySync::Types::Strict::Hash.default({}.freeze)
+    attribute :dataset_query, DatasetQuery
+
+    def self.native(database_id:,sql:,**kwargs)
+      new(database_id: database_id, dataset_query: DatasetQuery.native(sql: sql, database_id: database_id), **kwargs)
+    end
+
+    # @param put_card_request [PutCardRequest]
+    def self.from_request(put_card_request)
+      new(put_card_request.to_h.merge(database_id: put_card_request.dataset_query.database))
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/collection.rb

@@ -0,0 +1,12 @@
+require 'dry-struct'
+
+class MetabaseQuerySync::MetabaseApi
+  class Collection < Model
+    KEY = 'collection'
+
+    has :id, :archived, :name, :description
+    attribute :slug, MetabaseQuerySync::Types::Strict::String
+    attribute :location, MetabaseQuerySync::Types::Strict::String
+    attribute :parent_id, MetabaseQuerySync::Types::Strict::Integer.optional
+  end
+end

data/lib/metabase_query_sync/metabase_api/database.rb

@@ -0,0 +1,8 @@
+require 'dry-struct'
+
+class MetabaseQuerySync::MetabaseApi
+  class Database < Model
+    attribute :id, MetabaseQuerySync::Types::Strict::Integer
+    attribute :name, MetabaseQuerySync::Types::Strict::String
+  end
+end

data/lib/metabase_query_sync/metabase_api/faraday_metabase_api.rb

@@ -0,0 +1,118 @@
+require 'faraday'
+require 'faraday_middleware'
+require 'dry-monads'
+
+class MetabaseQuerySync::MetabaseApi
+  class FaradayMetabaseApi < self
+    include Dry::Monads[:result]
+
+    # @param client [Faraday::Connection]
+    # @param user [String]
+    # @param pass [String]
+    def initialize(client:, user:, pass:)
+      @client = client
+      @user = user
+      @pass = pass
+    end
+
+    # @param creds [MetabaseQuerySync::MetabaseCredentials]
+    def self.from_metabase_credentials(creds, &configure_faraday)
+      client = Faraday.new(url: creds.host) do |c|
+        c.request :json, content_type: /\bjson$/
+        c.response :json, content_type: /\bjson$/
+        c.request :url_encoded, content_type: /x-www-form-urlencoded/
+        # c.response :logger
+        c.adapter Faraday.default_adapter
+        c.headers['User-Agent'] =
+          "MetabaseQuerySync/#{MetabaseQuerySync::VERSION} (#{RUBY_ENGINE}#{RUBY_VERSION})"
+
+        configure_faraday.call(c) if configure_faraday
+      end
+
+      new(client: client, user: creds.user, pass: creds.pass)
+    end
+
+    def search(q, model: nil)
+      request(:get, '/api/search') do |req|
+        req.params.update(q: q, model: model)
+      end.fmap to_collection_of(Item)
+    end
+
+    def get_collection(id)
+      request(:get, "/api/collection/#{id}").fmap to(Collection)
+    end
+
+    def get_collection_items(collection_id)
+      request(:get, "/api/collection/#{collection_id}/items").fmap to_collection_of(Item)
+    end
+
+    def put_collection(collection_request)
+      if collection_request.id
+        request(:put, "/api/collection/#{collection_request.id}", body: collection_request.to_h)
+      else
+        request(:post, "/api/collection", body: collection_request.to_h)
+      end.fmap to(Collection)
+    end
+
+    def get_databases
+      request(:get, '/api/database').fmap to_collection_of(Database)
+    end
+
+    def get_card(id)
+      request(:get, "/api/card/#{id}").fmap to(Card)
+    end
+
+    def put_card(card_request)
+      if card_request.id
+        request(:put, "/api/card/#{card_request.id}", body: card_request.to_h)
+      else
+        request(:post, "/api/card", body: card_request.to_h)
+      end.fmap to(Card)
+    end
+
+    def get_pulse(id)
+      request(:get, "/api/pulse/#{id}").fmap to(Pulse)
+    end
+
+    def put_pulse(pulse_request)
+      if pulse_request.id
+        request(:put, "/api/pulse/#{pulse_request.id}", body: pulse_request.to_h)
+      else
+        request(:post, "/api/pulse", body: pulse_request.to_h)
+      end.fmap to(Pulse)
+    end
+
+    private
+
+    def to(klass)
+      klass.method(:new)
+    end
+
+    def to_collection_of(klass)
+      ->(collection) { collection.map(&klass.method(:new)) }
+    end
+
+    def token
+      @token ||= login.value!.id
+    end
+
+    def login
+      return request(:post, '/api/session', body: {
+        username: @user,
+        password: @pass
+      }, skip_token: true).fmap &to(Session)
+    end
+
+    def request(method, path, body: nil, headers: nil, skip_token: false, &block)
+      res = @client.run_request(method, path, body, headers) do |req|
+        req.headers['X-Metabase-Session'] = token unless skip_token
+        block.call(req) if block
+      end
+      if (200..299) === res.status
+        Success(res.body)
+      else
+        Failure(res)
+      end
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/item.rb

@@ -0,0 +1,20 @@
+require 'dry-struct'
+
+class MetabaseQuerySync::MetabaseApi
+  class Item < Model
+    attribute :id, MetabaseQuerySync::Types::Strict::Integer
+    attribute :name, MetabaseQuerySync::Types::Strict::String
+    attribute :description, MetabaseQuerySync::Types::Strict::String.optional.default(nil)
+    attribute :model, MetabaseQuerySync::Types::Strict::String
+
+    def card?
+      model == Card::KEY
+    end
+    def collection?
+      model == Collection::KEY
+    end
+    def pulse?
+      model == Pulse::KEY
+    end
+  end
+end

data/lib/metabase_query_sync/metabase_api/model.rb

@@ -0,0 +1,23 @@
+require 'dry-struct'
+class MetabaseQuerySync::MetabaseApi::Model < Dry::Struct
+  transform_keys &:to_sym
+
+  def self.has(*args)
+    args.each do |sym|
+      case sym
+      when :id
+        attribute :id, MetabaseQuerySync::Types::Strict::Integer.optional.default(nil)
+      when :archived
+        attribute :archived, MetabaseQuerySync::Types::Strict::Bool.default(false)
+      when :name
+        attribute :name, MetabaseQuerySync::Types::Strict::String
+      when :description
+        attribute :description, MetabaseQuerySync::Types::Strict::String.optional.default(nil)
+      when :collection_id
+        attribute :collection_id, MetabaseQuerySync::Types::Strict::Integer.optional.default(nil)
+      else
+        raise "Unexpected field for model (#{sym})"
+      end
+    end
+  end
+end