metabase_query_sync 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 020bfbb2b8958b1f6f3ed8067628bfb4942ddc8a751abc787b797c562b1500d8
-  data.tar.gz: ed21dd175d50d34dec40ae0fac3c29b65da668cba7d80c23bcb93d4b527d0a0d
+  metadata.gz: 196f3dcb53519b8a0dbbdab05d3a237181dc5e381e83e0a9627fe41e0ffa0024
+  data.tar.gz: e0c49e7710a5fbd18cb51ba1749700ed4e74503c6c0d3f63f786383380fbfa56
 SHA512:
-  metadata.gz: 6b41d731113fea1a55c85875b762a777d5ce77a27dc6efe9f5088ce22febbbc5a5beccc4cf12260a782a6b4e67fddcc7961941d3be1b747fee5ab747c5ca753a
-  data.tar.gz: 6b646e18d35d39a5da020fa43b3f0d9dbc9e9782fcb440a98371960f8866e659b266ad559f7710acc12de0bc21c9fb7e6176e0fcd09a6cc35c102be5c6ca0634
+  metadata.gz: e548655a3ad94a845b5966e248ba4376d0b98676fdd2afd13e94cf84b72fb3cd263df7d13147ea3382255df1b6643e92662ba0842996af04edf4cb667820f8bb
+  data.tar.gz: 4fc3a03c4c4e434982d0832d37ad0062c4322a90352636696ddced4f343501d56dcd4a492f97936b74cca5c9a796e88b7ef937e8b921fc7b2e73efb95a7bfc84

data/README.md

@@ -31,7 +31,7 @@ Build files with `.query.yaml` or `.pulse.yaml` suffix and sync those files up t
 name: Low Volume Orders
 sql: 'select * from orders'
 database: Local DB # must match name of database in metabase
-pulse: Hourly # must match local pulse name field, throws exception if no pulse is found with that name
+pulse: hourly # must match local pulse id (name of file), throws exception if no pulse is found with that name
 ```
 
 ```yaml
@@ -50,48 +50,97 @@ alerts:
       day: mon # first 3 character of day only needed if weekly
 ```
 
+### Understanding Ids
+
+Every metabase model defined in a file has an id which defaults to the name of the file minus the query/pulse.yaml suffix. An id can be explicitly set in the file if wanted. All references within files to other files must use the id of the item to reference.
+
+Given the following folder layout:
+
+```
+queries/
+  low-volume.query.yaml
+  catalog/
+    products-missing-images.query.yaml
+  hourly.pulse.yaml
+```
+
+The generated ids would be:
+
+```
+low-volume
+catalog/products-missing-images
+hourly
+```
+
 ### Running the Sync
 
 Then using the metabase-query-sync cli tool, you can sync those files directly into metabase:
 
 ```bash
 Command:
-  metabase-query-sync sync
+  metabase-query-sync s
 
 Usage:
-  metabase-query-sync sync ROOT_COLLECTION_ID PATH
+  metabase-query-sync s ROOT_COLLECTION_ID [PATHS]
 
 Description:
   Sync queries/pulses to your metabase root collection
 
 Arguments:
   ROOT_COLLECTION_ID  	# REQUIRED The root collection id to sync all items under.
-  PATH                	# REQUIRED The path to metabase item files to sync from.
+  PATHS               	# The paths to metabase item files to sync from. Support for scoped paths with custom_name:/path/to/folder is supported as well to ensure each imported item is scoped with custom_name.
 
 Options:
   --[no-]dry-run, -d              	# Perform a dry run and do not actually sync to the metabase instance., default: false
   --host=VALUE, -H VALUE          	# Metabase Host, if not set, will read from env at METABASE_QUERY_SYNC_HOST
   --user=VALUE, -u VALUE          	# Metabase User, if not set, will read from env at METABASE_QUERY_SYNC_USER
   --pass=VALUE, -p VALUE          	# Metabase Password, if not set, will read from env at METABASE_QUERY_SYNC_PASS
+  --config-file=VALUE, -f VALUE   	# explicit path to .metabase-query-sync.erb.yaml file in case its not in the working directory
   --help, -h                      	# Print this help
 ```
 
+### Using .metabase-query-sync.erb.yaml
+
+It's nice to configure the different paths to search for at once instead of configuring into the command each time, and for that, we support setting defalt config in a file named `.metabase-query-sync.yaml` which should be in the same working directory of the command executation.
+
+Here's an example file:
+
+```yaml
+credentials:
+  host: http://metabase:3000
+  user: ragboyjr@icloud.com
+  pass: <%= ENV["METABASE_PASS"] %>
+paths:
+  - 'sales:app/sales/queries'
+  - 'catalog:app/catalog/queries'
+```
+
+### Results in Metabase
+
+Navigating to metabase under the root collection provided, should show the synced queries and pulses!
+
+![All Synced Items](docs/img/readme-1-everything.png)
+![Card](docs/img/readme-1-card.png)
+![Pulse](docs/img/readme-1-pulse.png)
+
 ## Development
 
 - Install gems with `bundle install`
 - Run tests with `bundle exec rspec`
 
-### TODO
-
-- Support Collections and Syncing with collections
-- Matching IR vs MetabaseApi items should go off of the file name + collection id instead of just the name
 
-## Debugging with Metabase
+### Debugging with Metabase
 
 To setup the local data source for metabase, run `make db`.
 
 Starting the metabase docker container should automatically initialize an empty metabase installation with the main admin user account (ragboyjr@icloud.com / password123).
 
+## Roadmap
+
+- Support syncing from multiple paths with id prefixes
+  - e.g. /path-to-sales-queries:sales /path-to-payment-queries:payment
+- Support `.metabase-query-sync.erb.yaml` configuration file
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/ragboyjr/metabase-query-sync. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/ragboyjr/metabase-query-sync/blob/master/CODE_OF_CONDUCT.md).

data/lib/metabase_query_sync/cli.rb

@@ -14,16 +14,20 @@ module MetabaseQuerySync
       desc 'Sync queries/pulses to your metabase root collection'
 
       argument :root_collection_id, type: :integer, required: true, desc: 'The root collection id to sync all items under.'
-      argument :path, type: :string, required: true, desc: 'The path to metabase item files to sync from.'
+      argument :paths, type: :array, required: false, desc: 'The paths to metabase item files to sync from. Support for scoped paths with custom_name:/path/to/folder is supported as well to ensure each imported item is scoped with custom_name.'
       option :dry_run, type: :boolean, default: false, aliases: ['-d'], desc: 'Perform a dry run and do not actually sync to the metabase instance.'
       option :host, type: :string, aliases: ['-H'], desc: 'Metabase Host, if not set, will read from env at METABASE_QUERY_SYNC_HOST'
       option :user, type: :string, aliases: ['-u'], desc: 'Metabase User, if not set, will read from env at METABASE_QUERY_SYNC_USER'
       option :pass, type: :string, aliases: ['-p'], desc: 'Metabase Password, if not set, will read from env at METABASE_QUERY_SYNC_PASS'
+      option :config_file, type: :string, aliases: ['-f'], desc: 'explicit path to .metabase-query-sync.erb.yaml file in case its not in the working directory'
 
-      def call(root_collection_id:, path:, dry_run: false, host: nil, user: nil, pass: nil)
-        config = MetabaseQuerySync::Config.new(
-          credentials: MetabaseQuerySync::MetabaseCredentials.from_env(host: host, user: user, pass: pass),
-          path: path,
+      def call(root_collection_id:, paths: nil, dry_run: false, host: nil, user: nil, pass: nil, config_file: nil, **)
+        config = MetabaseQuerySync::Config.from_file(
+          config_file || File.join(Dir.pwd, '.metabase-query-sync.erb.yaml'),
+          paths: paths,
+          host: host,
+          user: user,
+          pass: pass,
         )
         sync = MetabaseQuerySync::Sync.from_config(config, Logger.new(STDOUT))
         sync.(MetabaseQuerySync::SyncRequest.new(root_collection_id: root_collection_id.to_i, dry_run: dry_run))

data/lib/metabase_query_sync/config.rb

@@ -1,12 +1,39 @@
+require 'dry-schema'
+require 'yaml'
+require 'erb'
+
 module MetabaseQuerySync
   class Config
-    attr_reader :credentials, :path
+    attr_reader :credentials, :paths
 
     # @param credentials [MetabaseQuerySync::MetabaseCredentials]
-    # @param path [String]
-    def initialize(credentials:, path:)
+    # @param paths [Array<String>]
+    def initialize(credentials:, paths:)
       @credentials = credentials
-      @path = path
+      @paths = paths
+    end
+
+    def self.from_file(path, paths: [], host: nil, user: nil, pass: nil)
+      if File.exists? path
+        data = YAML.load(ERB.new(File.read(path)).result)
+        result = Dry::Schema.JSON do
+          required(:paths).value(array[:string], min_size?: 1)
+          required(:credentials).hash do
+            required(:host).filled(:string)
+            required(:user).filled(:string)
+            required(:pass).filled(:string)
+          end
+        end.(data)
+        raise "Invalid data provided in config file: #{result.errors.to_h}" if result.failure?
+      end
+      new(
+        credentials: MetabaseCredentials.new(
+          host: host || data["credentials"]["host"],
+          user: user || data["credentials"]["user"],
+          pass: pass || data["credentials"]["pass"]
+        ),
+        paths: paths.empty? ? data["paths"] : paths
+      )
     end
   end
 end

data/lib/metabase_query_sync/ir/graph.rb

@@ -36,27 +36,17 @@ module MetabaseQuerySync::IR
       end)
     end
 
-    # @return [Query, nil]
-    def query_by_name(name)
-      queries.filter { |query| strcmp(query.name, name) }.first
-    end
-
-    # @return [Pulse, nil]
-    def pulse_by_name(name)
-      pulses.filter { |pulse| strcmp(pulse.name, name) }.first
-    end
-
     # @return [Array<Query>]
-    def queries_by_pulse(pulse_name)
-      queries.filter { |query| strcmp(query.pulse, pulse_name) }
+    def queries_by_pulse(pulse_id)
+      queries.filter { |query| query.pulse == pulse_id }
     end
 
     private
 
     def assert_traversal
-      pulse_names = pulses.map(&:name).map(&:downcase).to_set
+      pulse_ids = pulses.map(&:id).to_set
       queries.each do |q|
-        raise "No pulse (#{q.pulse}) found for query (#{q.name})" unless pulse_names === q.pulse.downcase
+        raise "No pulse (#{q.pulse}) found for query (#{q.name})" unless pulse_ids === q.pulse
       end
     end
 

data/lib/metabase_query_sync/ir/pulse.rb

@@ -22,11 +22,13 @@ module MetabaseQuerySync::IR
       attribute :schedule, Schedule
     end
 
+    attribute :id, string
     attribute :name, string
     attribute :skip_if_empty, bool.default(true)
     attribute :alerts, array.of(Alert)
 
     validate_with_schema do
+      required(:id).filled(:string)
       required(:name).filled(:string)
       optional(:skip_if_empty).value(:bool)
       required(:alerts).value(:array, min_size?: 1).each do

data/lib/metabase_query_sync/ir/query.rb

@@ -2,6 +2,7 @@ require 'dry-schema'
 
 module MetabaseQuerySync::IR
   class Query < Model
+    attribute :id, string
     attribute :name, string
     attribute :description, string.optional.default(nil)
     attribute :sql, string
@@ -10,6 +11,7 @@ module MetabaseQuerySync::IR
     attribute :collection, string.optional.default(nil)
 
     validate_with_schema do
+      required(:id).filled(:string)
       required(:name).filled(:string)
       required(:sql).filled(:string)
       required(:database).filled(:string)

data/lib/metabase_query_sync/metabase_state.rb

@@ -65,16 +65,6 @@ module MetabaseQuerySync
       collections.empty? && cards.empty? && pulses.empty?
     end
 
-    # @return [MetabaseApi::Pulse, nil]
-    def pulse_by_name(name)
-      pulses.filter { |p| p.name.downcase == name.downcase }.first
-    end
-
-    # @return [MetabaseApi::Card, nil]
-    def card_by_name(name)
-      cards.filter { |c| c.name.downcase == name.downcase }.first
-    end
-
     def database_by_name(name)
       databases.filter { |d| d.name.downcase == name.downcase }.first
     end

data/lib/metabase_query_sync/read_ir/from_files.rb

@@ -1,20 +1,50 @@
+require 'logger'
 require 'yaml'
 
 class MetabaseQuerySync::ReadIR
   class FromFiles < self
-    def initialize(path)
-      @path = path
+    def initialize(path, logger = nil)
+      @paths = path.is_a?(Array) ? path : [path]
+      @logger = logger || Logger.new(IO::NULL)
+
+      raise 'Paths must not be empty when reading from files' if @paths.empty?
     end
 
     def call
       MetabaseQuerySync::IR::Graph.from_items(
-        # @type [String] f
-        Dir[File.join(@path, "**/*.{query,pulse}.yaml")].map do |f|
-          data = YAML.load_file(f)
-          next MetabaseQuerySync::IR::Query.from_h(data) if f.end_with? 'query.yaml'
-          next MetabaseQuerySync::IR::Pulse.from_h(data) if f.end_with? 'pulse.yaml'
-        end
+        @paths.flat_map { |p| ir_items_from_path(p) }
       )
     end
+
+    private
+
+    # @param path [String]
+    def ir_items_from_path(path)
+      (scope, path) = split_path(path)
+      @logger.info "Reading IR Items from path (#{path}) and scope (#{scope})"
+
+      # @type [String] f
+      Dir[File.join(path, "**/*.{query,pulse}.yaml")].map do |f|
+        data = YAML.load_file(f)
+        next MetabaseQuerySync::IR::Query.from_h(prefix_id(scope, {"id" => id_from_file(path, f)}.merge(data))) if f.end_with? 'query.yaml'
+        next MetabaseQuerySync::IR::Pulse.from_h(prefix_id(scope, {"id" => id_from_file(path, f)}.merge(data))) if f.end_with? 'pulse.yaml'
+      end
+    end
+
+    # @param path [String]
+    def split_path(path)
+      path.include?(':') ? path.split(':', 2) : [nil, path]
+    end
+
+    # @param file_path [String]
+    def id_from_file(base_path, file_path)
+      file_path.gsub(/^#{Regexp.quote(File.join(base_path, ''))}(.+)\.(query|pulse)\.yaml$/, '\1')
+    end
+
+    # @param attributes [Hash]
+    def prefix_id(scope, attributes)
+      return attributes unless scope
+      attributes.merge({"id" => File.join(scope, attributes["id"])})
+    end
   end
 end

data/lib/metabase_query_sync/sync.rb

@@ -12,7 +12,7 @@ module MetabaseQuerySync
 
     # @param config [Config]
     def self.from_config(config, logger = nil)
-      new(ReadIR::FromFiles.new(config.path), MetabaseApi::FaradayMetabaseApi.from_metabase_credentials(config.credentials), logger)
+      new(ReadIR::FromFiles.new(config.paths, logger), MetabaseApi::FaradayMetabaseApi.from_metabase_credentials(config.credentials), logger)
     end
 
     # @param sync_req [SyncRequest]
@@ -48,7 +48,7 @@ module MetabaseQuerySync
     # @param metabase_state [MetabaseState]
     def delete_pulses(graph, metabase_state)
       metabase_state.pulses
-        .filter { |pulse| graph.pulse_by_name(pulse.name) == nil }
+        .filter { |pulse| find_graph_pulse(graph, id(pulse)) == nil }
         .map { |pulse| MetabaseApi::PutPulseRequest.from_pulse(pulse).new(archived: true) }
     end
 
@@ -56,7 +56,7 @@ module MetabaseQuerySync
     # @param metabase_state [MetabaseState]
     def delete_cards(graph, metabase_state)
       metabase_state.cards
-        .filter { |card| graph.query_by_name(card.name) == nil }
+        .filter { |card| find_graph_query(graph, id(card)) == nil }
         .map { |card| MetabaseApi::PutCardRequest.from_card(card).new(archived: true) }
     end
 
@@ -66,11 +66,12 @@ module MetabaseQuerySync
     def add_cards(graph, metabase_state, root_collection_id)
       graph.queries
         .map do |q|
-          [q, metabase_state.card_by_name(q.name)]
+          [q, find_api_card(metabase_state, id(q))]
         end
         .filter do |(q, card)|
           next true unless card
           card.dataset_query.native.query != q.sql ||
+            card.name != api_item_name(q) ||
             card.database_id != metabase_state.database_by_name(q.database)&.id ||
             card.description != q.description
         end
@@ -81,7 +82,7 @@ module MetabaseQuerySync
             id: card&.id,
             sql: q.sql,
             database_id: metabase_state.database_by_name(q.database)&.id,
-            name: q.name,
+            name: api_item_name(q),
             description: q.description,
             collection_id: root_collection_id,
           )
@@ -94,11 +95,11 @@ module MetabaseQuerySync
     def add_pulses(graph, metabase_state, root_collection_id)
       graph.pulses
         .map do |pulse|
-          api_pulse = metabase_state.pulse_by_name(pulse.name)
+          api_pulse = find_api_pulse(metabase_state, id(pulse))
           pulse_cards = graph
-            .queries_by_pulse(pulse.name)
+            .queries_by_pulse(pulse.id)
             .flat_map do |query|
-              card = metabase_state.card_by_name(query.name)
+              card = find_api_card(metabase_state, id(query))
               card ? [card] : []
             end
             .map { |card| MetabaseApi::Pulse::Card.new(id: card.id) }
@@ -125,12 +126,14 @@ module MetabaseQuerySync
         end
         .filter do |(pulse, api_pulse, pulse_cards, pulse_channels)|
           next true unless api_pulse
-          api_pulse.cards != pulse_cards || api_pulse.channels != pulse_channels
+          api_pulse.cards != pulse_cards ||
+            api_pulse.channels != pulse_channels ||
+            api_pulse.name != api_item_name(pulse)
         end
         .map do |(pulse, api_pulse, pulse_cards, pulse_channels)|
           MetabaseApi::PutPulseRequest.new(
             id: api_pulse&.id,
-            name: pulse.name,
+            name: "#{pulse.id}:#{pulse.name}",
             cards: pulse_cards,
             channels: pulse_channels,
             collection_id: root_collection_id,
@@ -163,5 +166,52 @@ module MetabaseQuerySync
         end
       end
     end
+
+    # @param item [IR::Model]
+    # @return [String]
+    def api_item_name(item)
+      "#{item.id}:#{item.name}"
+    end
+
+    # @param graph [IR::Graph]
+    # @param query_id [String]
+    # @return [IR::Query, nil]
+    def find_graph_query(graph, query_id)
+      graph.queries.filter { |q| id(q) == query_id }.first
+    end
+
+    # @param graph [IR::Graph]
+    # @param pulse_id [String]
+    # @return [IR::Pulse, nil]
+    def find_graph_pulse(graph, pulse_id)
+      graph.pulses.filter { |p| id(p) == pulse_id }.first
+    end
+
+    # @param metabase_state [MetabaseState]
+    # @param card_id [String]
+    # @return [MetabaseApi::Card, nil]
+    def find_api_card(metabase_state, card_id)
+      metabase_state.cards.filter { |c| id(c) == card_id }.first
+    end
+
+    # @param metabase_state [MetabaseState]
+    # @param pulse_id [String]
+    # @return [MetabaseApi::Pulse, nil]
+    def find_api_pulse(metabase_state, pulse_id)
+      metabase_state.pulses.filter { |p| id(p) == pulse_id }.first
+    end
+
+    # gets the normalized id from the given object to be used for comparisons
+    # @return [String]
+    def id(object)
+      case object
+      when IR::Model
+        object.id
+      when MetabaseApi::Model
+        object.name[/^([^:]+):/, 1] # metabase api names are constructed with #{IR id}:#{IR name}
+      else
+        raise "Unexpected object (#{object.class}) provided."
+      end
+    end
   end
 end

data/lib/metabase_query_sync/version.rb

@@ -1,3 +1,3 @@
 module MetabaseQuerySync
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metabase_query_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - RJ Garcia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-28 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zeitwerk