databricks 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c27151990dcc087e396f387f45f6909f54bb6eb8529b8835706ce84d3ac399ab
-  data.tar.gz: 0ddfaf4cd06981894bddc1ff2ede2bf2372aa6d08521699458502cdf4b83f308
+  metadata.gz: 06a826e00c327ae1b293c0da9f57632ec9103e128f253fcf2f8ce1229a64074c
+  data.tar.gz: 6830f2c2fc4fe380b92856f28d827e8573ccfbb78f86280d1d11269ccf602fbb
 SHA512:
-  metadata.gz: 747bf19db30cc24740b101aaacd14d22eeb7051995b7d1707917835a34fb90bc096304d2e11c18ea4c04c28049e5ead90961f1fcdb319642bbd54409feb580cf
-  data.tar.gz: dae0b046280172f770df37ce37440d0311f5e65896fe35209e3d0e900c4961d1a8396d195b570dac122c23b373aeb0be364f9c94178d54da888fab20dd81a819
+  metadata.gz: 601a0e36f9fb7bc79c9d4f42ecf9f3ff32eb1a531ce9d706f38b6731e3ef373da32b7cd2c404414e4747f1d2ec2b9833e0aa913ac8e1dac84f7e72fb8113d151
+  data.tar.gz: 7ef05a8b17ff9167e346cd129fbda117fd75289e9ac77d1b0679340b56bddcfec17d66c437a8c2fc95fa8278ee653a439936b408d1b7dacfde41aa69c85f8a36

data/lib/databricks.rb

@@ -1,5 +1,6 @@
-require 'databricks/domains/root'
+require 'databricks/connector'
 require 'databricks/resource'
+require 'databricks/resources/root'
 
 module Databricks
 
@@ -9,9 +10,9 @@ module Databricks
   # * *host* (String): Host to connect to
   # * *token* (String): Token to be used in th API
   # Result::
-  # * Domains::Root: The root domain of the API
+  # * Resources::Root: The root resource of the API
   def self.api(host, token)
-    Domains::Root.new(Resource.new(host, token))
+    Resources::Root.new(Connector.new(host, token))
   end
 
 end

data/lib/databricks/connector.rb

@@ -0,0 +1,79 @@
+require 'json'
+require 'rest-client'
+
+module Databricks
+
+  # Underlying connector making API calls
+  class Connector
+
+    # Constructor
+    #
+    # Parameters::
+    # * *host* (String): Host to connect to
+    # * *token* (String): Token to be used in th API
+    def initialize(host, token)
+      @host = host
+      @token = token
+    end
+
+    # Issue a GET request to the API with JSON payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # Result::
+    # * Object: JSON result
+    def get_json(api_path, json_payload = {})
+      JSON.parse(
+        RestClient::Request.execute(
+          method: :get,
+          url: "#{@host}/api/2.0/#{api_path}",
+          payload: json_payload.to_json,
+          headers: {
+            Authorization: "Bearer #{@token}",
+            'Content-Type': 'application/json'
+          }
+        ).body
+      )
+    end
+
+    # Issue a POST request to the API with JSON payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # Result::
+    # * Object: JSON result
+    def post_json(api_path, json_payload = {})
+      JSON.parse(
+        RestClient::Request.execute(
+          method: :post,
+          url: "#{@host}/api/2.0/#{api_path}",
+          payload: json_payload.to_json,
+          headers: {
+            Authorization: "Bearer #{@token}",
+            'Content-Type': 'application/json'
+          }
+        ).body
+      )
+    end
+
+    # Issue a POST request to the API with multipart form data payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *form_payload* (Hash): Form payload to include in the query [default = {}]
+    def post(api_path, form_payload = {})
+      RestClient::Request.execute(
+        method: :post,
+        url: "#{@host}/api/2.0/#{api_path}",
+        payload: form_payload.merge(multipart: true),
+        headers: {
+          Authorization: "Bearer #{@token}"
+        }
+      )
+    end
+
+  end
+
+end

data/lib/databricks/resource.rb

@@ -1,77 +1,95 @@
-require 'json'
-require 'rest-client'
+require 'forwardable'
 
 module Databricks
 
-  # Underlying resource making API calls
+  # Encapsulate a resource identified in the API.
+  # A resource can have some properties, directly accessible, and also gives access to eventual sub-resources to get a hierarchical organization of the API.
   class Resource
 
+    extend Forwardable
+
+    # Delegate the API low-level methods to the @connector object
+    def_delegators :@connector, *%i[
+      get_json
+      post
+      post_json
+    ]
+
+    # Declare sub-resources accessors.
+    # This will make sure this resource has methods named after the sub-resources identifiers.
+    #
+    # Parameters::
+    # * *resource_names* (Array<Symbol>): Resource names to instantiate
+    def self.sub_resources(*resource_names)
+      resource_names.flatten.each do |resource_name|
+        self.define_method(resource_name) do
+          sub_resource(resource_name)
+        end
+      end
+    end
+
     # Constructor
     #
     # Parameters::
-    # * *host* (String): Host to connect to
-    # * *token* (String): Token to be used in th API
-    def initialize(host, token)
-      @host = host
-      @token = token
+    # * *connector* (Connector): Connector handling API calls
+    def initialize(connector)
+      @connector = connector
+      # Keep a map of sub-resources instantiated, per resource name.
+      # Hash< Symbol, Resource >
+      @sub_resources = {}
+      # Properties linked to this resource
+      # Hash< Symbol, Object >
+      @properties = {}
     end
 
-    # Issue a GET request to the API with JSON payload
+    # Add/replace properties for this resource
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # * *properties* (Hash<Symbol,Object>): Properties for this resource
+    def add_properties(properties)
+      # Define getters for properties
+      (properties.keys - @properties.keys).each do |property_name|
+        if self.respond_to?(property_name)
+          raise "Can't define a property named #{property_name} - It's already used."
+        else
+          define_singleton_method(property_name) { @properties[property_name] }
+        end
+      end
+      @properties.merge!(properties)
+    end
+
+    # Return a simple string representation of this resource
+    #
     # Result::
-    # * Object: JSON result
-    def get_json(api_path, json_payload = {})
-      JSON.parse(
-        RestClient::Request.execute(
-          method: :get,
-          url: "#{@host}/api/2.0/#{api_path}",
-          payload: json_payload.to_json,
-          headers: {
-            Authorization: "Bearer #{@token}",
-            'Content-Type': 'application/json'
-          }
-        ).body
-      )
+    # * String: Default representation
+    def inspect
+      "#<#{self.class.name.split('::').last} - #{@properties}>"
     end
 
-    # Issue a POST request to the API with JSON payload
+    # Instantiate a sub-resource.
+    # Keep a cache of it.
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # * *resource_name* (Symbol): Resource name.
     # Result::
-    # * Object: JSON result
-    def post_json(api_path, json_payload = {})
-      JSON.parse(
-        RestClient::Request.execute(
-          method: :post,
-          url: "#{@host}/api/2.0/#{api_path}",
-          payload: json_payload.to_json,
-          headers: {
-            Authorization: "Bearer #{@token}",
-            'Content-Type': 'application/json'
-          }
-        ).body
-      )
+    # * Resource: Corresponding sub-resource
+    def sub_resource(resource_name)
+      @sub_resources[resource_name] = new_resource(resource_name) unless @sub_resources.key?(resource_name)
+      @sub_resources[resource_name]
     end
 
-    # Issue a POST request to the API with multipart form data payload
+    # Instantiate a new resource, with optional properties
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *form_payload* (Hash): Form payload to include in the query [default = {}]
-    def post(api_path, form_payload = {})
-      RestClient::Request.execute(
-        method: :post,
-        url: "#{@host}/api/2.0/#{api_path}",
-        payload: form_payload.merge(multipart: true),
-        headers: {
-          Authorization: "Bearer #{@token}"
-        }
-      )
+    # * *resource_name* (Symbol): The resource's name.
+    # * *properties* (Hash<Symbol or String,Object>): This resource's initial properties [default = {}]
+    # Result::
+    # * Resource: The corresponding resource
+    def new_resource(resource_name, properties = {})
+      require "#{__dir__}/resources/#{resource_name}.rb"
+      resource = Resources.const_get(resource_name.to_s.split('_').collect(&:capitalize).join.to_sym).new(@connector)
+      resource.add_properties(properties.transform_keys(&:to_sym))
+      resource
     end
 
   end

data/lib/databricks/resources/cluster.rb

@@ -0,0 +1,27 @@
+module Databricks
+
+  module Resources
+
+    class Cluster < Resource
+
+      # Edit properties of this cluster.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties of this cluster
+      def edit(**properties)
+        # Make sure we don't change its ID
+        properties[:cluster_id] = cluster_id
+        post_json('clusters/edit', properties)
+        add_properties(properties)
+      end
+
+      # Delete a cluster
+      def delete
+        post_json('clusters/delete', { cluster_id: cluster_id })
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/clusters.rb

@@ -0,0 +1,39 @@
+module Databricks
+
+  module Resources
+
+    # Provide the Clusters API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/clusters.html
+    class Clusters < Resource
+
+      # List clusters
+      #
+      # Result::
+      # * Array<Hash>: List of clusters information
+      def list
+        (get_json('clusters/list')['clusters'] || []).map { |properties| new_resource(:cluster, properties) }
+      end
+
+      # Get a cluster based on its cluster_id
+      #
+      # Result::
+      # * Cluster: The cluster
+      def get(cluster_id)
+        new_resource(:cluster, get_json('clusters/get', { cluster_id: cluster_id }))
+      end
+
+      # Create a new cluster.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties to create the cluster
+      def create(**properties)
+        cluster = new_resource(:cluster, post_json('clusters/create', properties))
+        cluster.add_properties(properties)
+        cluster
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/{domains → resources}/dbfs.rb

@@ -1,12 +1,10 @@
-require 'databricks/domain'
-
 module Databricks
 
-  module Domains
+  module Resources
 
     # Provide the DBFS API
     # cf. https://docs.databricks.com/dev-tools/api/latest/dbfs.html
-    class Dbfs < Domain
+    class Dbfs < Resource
 
       # List a path
       #
@@ -15,12 +13,7 @@ module Databricks
       # Result::
       # * Array<String>: List of DBFS paths
       def list(path)
-        @resource.get_json(
-          'dbfs/list',
-          {
-            path: path
-          }
-        )['files'].map { |file_info| file_info['path'] }
+        (get_json('dbfs/list', { path: path })['files'] || []).map { |properties| new_resource(:file, properties) }
       end
 
       # Put a new file
@@ -29,16 +22,31 @@ module Databricks
       # * *path* (String): Path to the file to create
       # * *local_file* (String): Path to the local file to put
       def put(path, local_file)
-        @resource.post(
+        post(
           'dbfs/put',
           {
             path: path,
-            contents: File.new(local_file, 'rb'),
+            contents: ::File.new(local_file, 'rb'),
             overwrite: true
           }
         )
       end
 
+      # Delete a path
+      #
+      # Parameters::
+      # * *path* (String): Path to delete
+      # * *recursive* (Boolean): Do we delete recursively? [default: false]
+      def delete(path, recursive: false)
+        post_json(
+          'dbfs/delete',
+          {
+            path: path,
+            recursive: recursive
+          }
+        )
+      end
+
     end
 
   end

data/lib/databricks/resources/file.rb

@@ -0,0 +1,11 @@
+module Databricks
+
+  module Resources
+
+    class File < Resource
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/job.rb

@@ -0,0 +1,11 @@
+module Databricks
+
+  module Resources
+
+    class Job < Resource
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/jobs.rb

@@ -0,0 +1,31 @@
+module Databricks
+
+  module Resources
+
+    # Provide the Jobs API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/jobs.html
+    class Jobs < Resource
+
+      # List jobs
+      #
+      # Result::
+      # * Array<Hash>: List of jobs information
+      def list
+        (get_json('jobs/list')['jobs'] || []).map { |properties| new_resource(:job, properties) }
+      end
+
+      # Create a new job.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties to create the job
+      def create(**properties)
+        job = new_resource(:job, post_json('jobs/create', properties))
+        job.add_properties(properties)
+        job
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/{domains → resources}/root.rb

@@ -1,14 +1,15 @@
-require 'databricks/domain'
+require 'databricks/resource'
 
 module Databricks
 
-  module Domains
+  module Resources
 
     # API entry point
     # cf. https://docs.databricks.com/dev-tools/api/latest/index.html
-    class Root < Domain
+    class Root < Resource
 
-      sub_domains %i[
+      sub_resources %i[
+        clusters
         dbfs
         jobs
       ]

data/lib/databricks/version.rb

@@ -1,5 +1,5 @@
 module Databricks
 
-  VERSION = '1.0.0'
+  VERSION = '2.0.0'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: databricks
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -74,11 +74,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/databricks.rb
-- lib/databricks/domain.rb
-- lib/databricks/domains/dbfs.rb
-- lib/databricks/domains/jobs.rb
-- lib/databricks/domains/root.rb
+- lib/databricks/connector.rb
 - lib/databricks/resource.rb
+- lib/databricks/resources/cluster.rb
+- lib/databricks/resources/clusters.rb
+- lib/databricks/resources/dbfs.rb
+- lib/databricks/resources/file.rb
+- lib/databricks/resources/job.rb
+- lib/databricks/resources/jobs.rb
+- lib/databricks/resources/root.rb
 - lib/databricks/version.rb
 homepage: https://github.com/Muriel-Salvan/databricks
 licenses:

data/lib/databricks/domain.rb

@@ -1,47 +0,0 @@
-module Databricks
-
-  # Encapsulate a part of the API for better organization
-  class Domain
-
-    # Declare sub-domain accessors in the current domain.
-    # This will make sure the current domain has methods named after the sub-domain identifiers that will instantiate the corresponding domains at will.
-    #
-    # Parameters::
-    # * *domains* (Array<Symbol>): Domains to instantiate
-    def self.sub_domains(*domains)
-      domains.flatten.each do |domain|
-        self.define_method(domain) do
-          sub_domain(domain)
-        end
-      end
-    end
-
-    # Instantiate a sub-domain.
-    # Keep a cache of it.
-    #
-    # Parameters::
-    # * *domain* (Symbol): Sub-domain identifier.
-    # Result::
-    # * Domain: Corresponding sub-domain
-    def sub_domain(domain)
-      unless @sub_domains.key?(domain)
-        require "#{__dir__}/domains/#{domain}.rb"
-        @sub_domains[domain] = Domains.const_get(domain.to_s.split('_').collect(&:capitalize).join.to_sym).new(@resource)
-      end
-      @sub_domains[domain]
-    end
-
-    # Constructor
-    #
-    # Parameters::
-    # * *resource* (Resource): Resource handling API calls
-    def initialize(resource)
-      @resource = resource
-      # Keep a map of sub-domains instantiated, per domain identifier.
-      # Hash< Symbol, Domain >
-      @sub_domains = {}
-    end
-
-  end
-
-end

data/lib/databricks/domains/jobs.rb

@@ -1,31 +0,0 @@
-require 'databricks/domain'
-
-module Databricks
-
-  module Domains
-
-    # Provide the Jobs API
-    # cf. https://docs.databricks.com/dev-tools/api/latest/jobs.html
-    class Jobs < Domain
-
-      # List a path
-      #
-      # Result::
-      # * Array<Hash>: List of jobs information
-      def list
-        @resource.get_json('jobs/list')['jobs']
-      end
-
-      # Create a new job
-      #
-      # Parameters::
-      # * *settings* (Hash<Symbol,Object>): Settings to create the job
-      def create(**settings)
-        @resource.post_json('jobs/create', settings)
-      end
-
-    end
-
-  end
-
-end