databricks 1.0.0 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c27151990dcc087e396f387f45f6909f54bb6eb8529b8835706ce84d3ac399ab
-  data.tar.gz: 0ddfaf4cd06981894bddc1ff2ede2bf2372aa6d08521699458502cdf4b83f308
+  metadata.gz: cc44c2e0ce0d40ceb08101541a9da02226b83200c681b675418baed111c06336
+  data.tar.gz: b32827444314a21c8453a0f620a32b66501e392edc620555b7f6dfd1558b536b
 SHA512:
-  metadata.gz: 747bf19db30cc24740b101aaacd14d22eeb7051995b7d1707917835a34fb90bc096304d2e11c18ea4c04c28049e5ead90961f1fcdb319642bbd54409feb580cf
-  data.tar.gz: dae0b046280172f770df37ce37440d0311f5e65896fe35209e3d0e900c4961d1a8396d195b570dac122c23b373aeb0be364f9c94178d54da888fab20dd81a819
+  metadata.gz: 638132f2c85b3a8a36505933db36bef16e81e093c42e93ac268becf5407a8291cb93add54efd941c3fdbb4ca08a165dc7351b03d48cebdcff226c94dad10f6ae
+  data.tar.gz: 4837f5936f68ed8e4746392b1954b9d40b12a099408a2f4d3397852b56c30a81ca8307e90898e1e72e381120df0faf47f7ad4bed7f605729befff2ce32cd01a2

data/lib/databricks.rb

@@ -1,5 +1,6 @@
-require 'databricks/domains/root'
+require 'databricks/connector'
 require 'databricks/resource'
+require 'databricks/resources/root'
 
 module Databricks
 
@@ -9,9 +10,9 @@ module Databricks
   # * *host* (String): Host to connect to
   # * *token* (String): Token to be used in th API
   # Result::
-  # * Domains::Root: The root domain of the API
+  # * Resources::Root: The root resource of the API
   def self.api(host, token)
-    Domains::Root.new(Resource.new(host, token))
+    Resources::Root.new(Connector.new(host, token))
   end
 
 end

data/lib/databricks/connector.rb

@@ -0,0 +1,79 @@
+require 'json'
+require 'rest-client'
+
+module Databricks
+
+  # Underlying connector making API calls
+  class Connector
+
+    # Constructor
+    #
+    # Parameters::
+    # * *host* (String): Host to connect to
+    # * *token* (String): Token to be used in th API
+    def initialize(host, token)
+      @host = host
+      @token = token
+    end
+
+    # Issue a GET request to the API with JSON payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # Result::
+    # * Object: JSON result
+    def get_json(api_path, json_payload = {})
+      JSON.parse(
+        RestClient::Request.execute(
+          method: :get,
+          url: "#{@host}/api/2.0/#{api_path}",
+          payload: json_payload.to_json,
+          headers: {
+            Authorization: "Bearer #{@token}",
+            'Content-Type': 'application/json'
+          }
+        ).body
+      )
+    end
+
+    # Issue a POST request to the API with JSON payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # Result::
+    # * Object: JSON result
+    def post_json(api_path, json_payload = {})
+      JSON.parse(
+        RestClient::Request.execute(
+          method: :post,
+          url: "#{@host}/api/2.0/#{api_path}",
+          payload: json_payload.to_json,
+          headers: {
+            Authorization: "Bearer #{@token}",
+            'Content-Type': 'application/json'
+          }
+        ).body
+      )
+    end
+
+    # Issue a POST request to the API with multipart form data payload
+    #
+    # Parameters::
+    # * *api_path* (String): API path to query
+    # * *form_payload* (Hash): Form payload to include in the query [default = {}]
+    def post(api_path, form_payload = {})
+      RestClient::Request.execute(
+        method: :post,
+        url: "#{@host}/api/2.0/#{api_path}",
+        payload: form_payload.merge(multipart: true),
+        headers: {
+          Authorization: "Bearer #{@token}"
+        }
+      )
+    end
+
+  end
+
+end

data/lib/databricks/resource.rb

@@ -1,77 +1,130 @@
-require 'json'
-require 'rest-client'
+require 'forwardable'
 
 module Databricks
 
-  # Underlying resource making API calls
+  # Encapsulate a resource identified in the API.
+  # A resource can have some properties, directly accessible, and also gives access to eventual sub-resources to get a hierarchical organization of the API.
   class Resource
 
+    extend Forwardable
+
+    # Delegate the API low-level methods to the @connector object
+    def_delegators :@connector, *%i[
+      get_json
+      post
+      post_json
+    ]
+
+    # Get an accessor on all properties of this resource
+    # Hash< Symbol, Object >
+    attr_reader :properties
+
+    # Declare sub-resources accessors.
+    # This will make sure this resource has methods named after the sub-resources identifiers.
+    #
+    # Parameters::
+    # * *resource_names* (Array<Symbol>): Resource names to instantiate
+    def self.sub_resources(*resource_names)
+      resource_names.flatten.each do |resource_name|
+        self.define_method(resource_name) do
+          sub_resource(resource_name)
+        end
+      end
+    end
+
     # Constructor
     #
     # Parameters::
-    # * *host* (String): Host to connect to
-    # * *token* (String): Token to be used in th API
-    def initialize(host, token)
-      @host = host
-      @token = token
+    # * *connector* (Connector): Connector handling API calls
+    def initialize(connector)
+      @connector = connector
+      # Keep a map of sub-resources instantiated, per resource name.
+      # Hash< Symbol, Resource >
+      @sub_resources = {}
+      # Properties linked to this resource
+      # Hash< Symbol, Object >
+      @properties = {}
     end
 
-    # Issue a GET request to the API with JSON payload
+    # Add/replace properties for this resource.
+    # Properties will be deep-symbolized.
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # * *properties* (Hash<Symbol or String,Object>): Properties for this resource
+    # * *replace* (Boolean): Should we replace properties instead of merging them? [default: false]
+    def add_properties(properties, replace: false)
+      symbolized_properties = deep_symbolize(properties)
+      # Define getters for properties
+      (symbolized_properties.keys - @properties.keys).each do |property_name|
+        if self.respond_to?(property_name)
+          raise "Can't define a property named #{property_name} - It's already used."
+        else
+          define_singleton_method(property_name) { @properties[property_name] }
+        end
+      end
+      if replace
+        @properties = symbolized_properties
+      else
+        @properties.merge!(symbolized_properties)
+      end
+    end
+
+    # Return a simple string representation of this resource
+    #
+    # Result::
+    # * String: Default representation
+    def inspect
+      "#<#{self.class.name.split('::').last} - #{@properties}>"
+    end
+
+    # Instantiate a sub-resource.
+    # Keep a cache of it.
+    #
+    # Parameters::
+    # * *resource_name* (Symbol): Resource name.
     # Result::
-    # * Object: JSON result
-    def get_json(api_path, json_payload = {})
-      JSON.parse(
-        RestClient::Request.execute(
-          method: :get,
-          url: "#{@host}/api/2.0/#{api_path}",
-          payload: json_payload.to_json,
-          headers: {
-            Authorization: "Bearer #{@token}",
-            'Content-Type': 'application/json'
-          }
-        ).body
-      )
+    # * Resource: Corresponding sub-resource
+    def sub_resource(resource_name)
+      @sub_resources[resource_name] = new_resource(resource_name) unless @sub_resources.key?(resource_name)
+      @sub_resources[resource_name]
     end
 
-    # Issue a POST request to the API with JSON payload
+    # Instantiate a new resource, with optional properties
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *json_payload* (Object): JSON payload to include in the query [default = {}]
+    # * *resource_name* (Symbol): The resource's name.
+    # * *properties* (Hash<Symbol or String,Object>): This resource's initial properties [default = {}]
     # Result::
-    # * Object: JSON result
-    def post_json(api_path, json_payload = {})
-      JSON.parse(
-        RestClient::Request.execute(
-          method: :post,
-          url: "#{@host}/api/2.0/#{api_path}",
-          payload: json_payload.to_json,
-          headers: {
-            Authorization: "Bearer #{@token}",
-            'Content-Type': 'application/json'
-          }
-        ).body
-      )
+    # * Resource: The corresponding resource
+    def new_resource(resource_name, properties = {})
+      require "#{__dir__}/resources/#{resource_name}.rb"
+      resource = Resources.const_get(resource_name.to_s.split('_').collect(&:capitalize).join.to_sym).new(@connector)
+      resource.add_properties(properties)
+      resource
     end
 
-    # Issue a POST request to the API with multipart form data payload
+    private
+
+    # Deep-symbolize a JSON object
     #
     # Parameters::
-    # * *api_path* (String): API path to query
-    # * *form_payload* (Hash): Form payload to include in the query [default = {}]
-    def post(api_path, form_payload = {})
-      RestClient::Request.execute(
-        method: :post,
-        url: "#{@host}/api/2.0/#{api_path}",
-        payload: form_payload.merge(multipart: true),
-        headers: {
-          Authorization: "Bearer #{@token}"
-        }
-      )
+    # * *json* (Object): The JSON object
+    # Result::
+    # * Object: Symbolized JSON object
+    def deep_symbolize(json)
+      case json
+      when Hash
+        Hash[json.map do |k, v|
+          [
+            k.is_a?(String) ? k.to_sym : k,
+            deep_symbolize(v)
+          ]
+        end]
+      when Array
+        json.map { |e| deep_symbolize(e) }
+      else
+        json
+      end
     end
 
   end

data/lib/databricks/resources/cluster.rb

@@ -0,0 +1,27 @@
+module Databricks
+
+  module Resources
+
+    class Cluster < Resource
+
+      # Edit properties of this cluster.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties of this cluster
+      def edit(**properties)
+        # Make sure we don't change its ID
+        properties[:cluster_id] = cluster_id
+        post_json('clusters/edit', properties)
+        add_properties(properties)
+      end
+
+      # Delete a cluster
+      def delete
+        post_json('clusters/delete', { cluster_id: cluster_id })
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/clusters.rb

@@ -0,0 +1,43 @@
+module Databricks
+
+  module Resources
+
+    # Provide the Clusters API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/clusters.html
+    class Clusters < Resource
+
+      # List clusters
+      #
+      # Result::
+      # * Array<Cluster>: List of clusters
+      def list
+        (get_json('clusters/list')['clusters'] || []).map { |properties| new_resource(:cluster, properties) }
+      end
+
+      # Get a cluster based on its cluster_id
+      #
+      # Parameters::
+      # * *cluster_id* (String): The cluster id to get
+      # Result::
+      # * Cluster: The cluster
+      def get(cluster_id)
+        new_resource(:cluster, get_json('clusters/get', { cluster_id: cluster_id }))
+      end
+
+      # Create a new cluster.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties to create the cluster
+      # Result::
+      # * Cluster: The new cluster created
+      def create(**properties)
+        cluster = new_resource(:cluster, post_json('clusters/create', properties))
+        cluster.add_properties(properties)
+        cluster
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/dbfs.rb

@@ -0,0 +1,78 @@
+require 'base64'
+
+module Databricks
+
+  module Resources
+
+    # Provide the DBFS API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/dbfs.html
+    class Dbfs < Resource
+
+      # List a path
+      #
+      # Parameters::
+      # * *path* (String): Path to be listed
+      # Result::
+      # * Array<String>: List of DBFS paths
+      def list(path)
+        (get_json('dbfs/list', { path: path })['files'] || []).map { |properties| new_resource(:file, properties) }
+      end
+
+      # Put a new file
+      #
+      # Parameters::
+      # * *path* (String): Path to the file to create
+      # * *local_file* (String): Path to the local file to put
+      def put(path, local_file)
+        post(
+          'dbfs/put',
+          {
+            path: path,
+            contents: ::File.new(local_file, 'rb'),
+            overwrite: true
+          }
+        )
+      end
+
+      # Delete a path
+      #
+      # Parameters::
+      # * *path* (String): Path to delete
+      # * *recursive* (Boolean): Do we delete recursively? [default: false]
+      def delete(path, recursive: false)
+        post_json(
+          'dbfs/delete',
+          {
+            path: path,
+            recursive: recursive
+          }
+        )
+      end
+
+      # Read a file.
+      # Decodes the content in the json response (that is originally Base64-encoded).
+      #
+      # Parameters::
+      # * *path* (String): Path to the file to read
+      # * *offset* (Integer): Offset to read from [default: 0]
+      # * *length* (Integer): Number of nytes to read from (max 1MB) [default: 524_288]
+      def read(path, offset: 0, length: 524_288)
+        raw_json = get_json(
+          'dbfs/read',
+          {
+            path: path,
+            offset: offset,
+            length: length
+          }
+        )
+        {
+          'bytes_read' => raw_json['bytes_read'],
+          'data' => Base64.decode64(raw_json['data'])
+        }
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/file.rb

@@ -0,0 +1,11 @@
+module Databricks
+
+  module Resources
+
+    class File < Resource
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/instance_pool.rb

@@ -0,0 +1,27 @@
+module Databricks
+
+  module Resources
+
+    class InstancePool < Resource
+
+      # Edit properties of this instance pool.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties of this cluster
+      def edit(**properties)
+        # Make sure we don't change its ID
+        properties[:instance_pool_id] = instance_pool_id
+        post_json('instance-pools/edit', properties)
+        add_properties(properties)
+      end
+
+      # Delete this instance pool
+      def delete
+        post_json('instance-pools/delete', { instance_pool_id: instance_pool_id })
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/instance_pools.rb

@@ -0,0 +1,43 @@
+module Databricks
+
+  module Resources
+
+    # Provide the Instance Pools API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/instance-pools.html
+    class InstancePools < Resource
+
+      # List instance pools
+      #
+      # Result::
+      # * Array<InstancePool>: List of instance pools
+      def list
+        (get_json('instance-pools/list')['instance_pools'] || []).map { |properties| new_resource(:instance_pool, properties) }
+      end
+
+      # Get an instance pool based on its instance_pool_id
+      #
+      # Parameters::
+      # * *instance_pool_id* (String): The instance pool id to get
+      # Result::
+      # * InstancePool: The instance pool
+      def get(instance_pool_id)
+        new_resource(:instance_pool, get_json('instance-pools/get', { instance_pool_id: instance_pool_id }))
+      end
+
+      # Create a new instance pool.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties to create the instance pool
+      # Result::
+      # * InstancePool: The new instance pool created
+      def create(**properties)
+        instance_pool = new_resource(:instance_pool, post_json('instance-pools/create', properties))
+        instance_pool.add_properties(properties)
+        instance_pool
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/job.rb

@@ -0,0 +1,32 @@
+module Databricks
+
+  module Resources
+
+    class Job < Resource
+
+      # Reset properties of this job.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): New job's properties
+      def reset(**properties)
+        # Make sure we don't change its ID
+        post_json(
+          'jobs/reset',
+          {
+            job_id: job_id,
+            new_settings: properties
+          }
+        )
+        add_properties(properties.merge(job_id: job_id), replace: true)
+      end
+
+      # Delete this job
+      def delete
+        post_json('jobs/delete', { job_id: job_id })
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/resources/jobs.rb

@@ -0,0 +1,47 @@
+module Databricks
+
+  module Resources
+
+    # Provide the Jobs API
+    # cf. https://docs.databricks.com/dev-tools/api/latest/jobs.html
+    class Jobs < Resource
+
+      # List jobs
+      #
+      # Result::
+      # * Array<Job>: List of jobs information
+      def list
+        (get_json('jobs/list')['jobs'] || []).map do |properties|
+          # The settings property should be merged at root
+          new_resource(:job, properties.merge(properties.delete('settings')))
+        end
+      end
+
+      # Get a job based on its job_id
+      #
+      # Parameters::
+      # * *job_id* (String): The job id to get
+      # Result::
+      # * Job: The job
+      def get(job_id)
+        properties = get_json('jobs/get', { job_id: job_id })
+        new_resource(:job, properties.merge(properties.delete('settings')))
+      end
+
+      # Create a new job.
+      #
+      # Parameters::
+      # * *properties* (Hash<Symbol,Object>): Properties to create the job
+      # Result::
+      # * Job: The new job created
+      def create(**properties)
+        job = new_resource(:job, post_json('jobs/create', properties))
+        job.add_properties(properties)
+        job
+      end
+
+    end
+
+  end
+
+end

data/lib/databricks/{domains → resources}/root.rb

@@ -1,15 +1,17 @@
-require 'databricks/domain'
+require 'databricks/resource'
 
 module Databricks
 
-  module Domains
+  module Resources
 
     # API entry point
     # cf. https://docs.databricks.com/dev-tools/api/latest/index.html
-    class Root < Domain
+    class Root < Resource
 
-      sub_domains %i[
+      sub_resources %i[
+        clusters
         dbfs
+        instance_pools
         jobs
       ]
 

data/lib/databricks/version.rb

@@ -1,5 +1,5 @@
 module Databricks
 
-  VERSION = '1.0.0'
+  VERSION = '2.3.1'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: databricks
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.3.1
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -74,11 +74,17 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/databricks.rb
-- lib/databricks/domain.rb
-- lib/databricks/domains/dbfs.rb
-- lib/databricks/domains/jobs.rb
-- lib/databricks/domains/root.rb
+- lib/databricks/connector.rb
 - lib/databricks/resource.rb
+- lib/databricks/resources/cluster.rb
+- lib/databricks/resources/clusters.rb
+- lib/databricks/resources/dbfs.rb
+- lib/databricks/resources/file.rb
+- lib/databricks/resources/instance_pool.rb
+- lib/databricks/resources/instance_pools.rb
+- lib/databricks/resources/job.rb
+- lib/databricks/resources/jobs.rb
+- lib/databricks/resources/root.rb
 - lib/databricks/version.rb
 homepage: https://github.com/Muriel-Salvan/databricks
 licenses:
@@ -99,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: Rubygem wrapping the Databricks REST API

data/lib/databricks/domain.rb

@@ -1,47 +0,0 @@
-module Databricks
-
-  # Encapsulate a part of the API for better organization
-  class Domain
-
-    # Declare sub-domain accessors in the current domain.
-    # This will make sure the current domain has methods named after the sub-domain identifiers that will instantiate the corresponding domains at will.
-    #
-    # Parameters::
-    # * *domains* (Array<Symbol>): Domains to instantiate
-    def self.sub_domains(*domains)
-      domains.flatten.each do |domain|
-        self.define_method(domain) do
-          sub_domain(domain)
-        end
-      end
-    end
-
-    # Instantiate a sub-domain.
-    # Keep a cache of it.
-    #
-    # Parameters::
-    # * *domain* (Symbol): Sub-domain identifier.
-    # Result::
-    # * Domain: Corresponding sub-domain
-    def sub_domain(domain)
-      unless @sub_domains.key?(domain)
-        require "#{__dir__}/domains/#{domain}.rb"
-        @sub_domains[domain] = Domains.const_get(domain.to_s.split('_').collect(&:capitalize).join.to_sym).new(@resource)
-      end
-      @sub_domains[domain]
-    end
-
-    # Constructor
-    #
-    # Parameters::
-    # * *resource* (Resource): Resource handling API calls
-    def initialize(resource)
-      @resource = resource
-      # Keep a map of sub-domains instantiated, per domain identifier.
-      # Hash< Symbol, Domain >
-      @sub_domains = {}
-    end
-
-  end
-
-end

data/lib/databricks/domains/dbfs.rb

@@ -1,46 +0,0 @@
-require 'databricks/domain'
-
-module Databricks
-
-  module Domains
-
-    # Provide the DBFS API
-    # cf. https://docs.databricks.com/dev-tools/api/latest/dbfs.html
-    class Dbfs < Domain
-
-      # List a path
-      #
-      # Parameters::
-      # * *path* (String): Path to be listed
-      # Result::
-      # * Array<String>: List of DBFS paths
-      def list(path)
-        @resource.get_json(
-          'dbfs/list',
-          {
-            path: path
-          }
-        )['files'].map { |file_info| file_info['path'] }
-      end
-
-      # Put a new file
-      #
-      # Parameters::
-      # * *path* (String): Path to the file to create
-      # * *local_file* (String): Path to the local file to put
-      def put(path, local_file)
-        @resource.post(
-          'dbfs/put',
-          {
-            path: path,
-            contents: File.new(local_file, 'rb'),
-            overwrite: true
-          }
-        )
-      end
-
-    end
-
-  end
-
-end

data/lib/databricks/domains/jobs.rb

@@ -1,31 +0,0 @@
-require 'databricks/domain'
-
-module Databricks
-
-  module Domains
-
-    # Provide the Jobs API
-    # cf. https://docs.databricks.com/dev-tools/api/latest/jobs.html
-    class Jobs < Domain
-
-      # List a path
-      #
-      # Result::
-      # * Array<Hash>: List of jobs information
-      def list
-        @resource.get_json('jobs/list')['jobs']
-      end
-
-      # Create a new job
-      #
-      # Parameters::
-      # * *settings* (Hash<Symbol,Object>): Settings to create the job
-      def create(**settings)
-        @resource.post_json('jobs/create', settings)
-      end
-
-    end
-
-  end
-
-end