ksql 0.1.0.beta

data/examples/quickstart.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'ksql'
+
+# The following example is from https://ksqldb.io/quickstart.html
+
+# * Create a stream
+
+Ksql::Client.ksql("CREATE STREAM riderLocations (profileId VARCHAR, latitude DOUBLE, longitude DOUBLE)
+  WITH (kafka_topic='locations', value_format='json', partitions=1);")
+
+# * Create materialized views
+
+Ksql::Client.ksql("CREATE TABLE currentLocation AS
+  SELECT profileId,
+         LATEST_BY_OFFSET(latitude) AS la,
+         LATEST_BY_OFFSET(longitude) AS lo
+  FROM riderlocations
+  GROUP BY profileId
+  EMIT CHANGES;")
+
+Ksql::Client.ksql("CREATE TABLE ridersNearMountainView AS
+  SELECT ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1) AS distanceInMiles,
+         COLLECT_LIST(profileId) AS riders,
+         COUNT(*) AS count
+  FROM currentLocation
+  GROUP BY ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1);")
+
+# * Run a push query over the stream
+
+stream = Ksql::Client.stream('SELECT * FROM riderLocations WHERE GEO_DISTANCE(latitude, longitude, 37.4133, -122.1162) <= 5 EMIT CHANGES;')
+
+stream.start do |location|
+  File.open('output.log', 'a') do |f|
+    f.write("Latitude: #{location.latitude}, Longitude: #{location.longitude}")
+  end
+end
+
+# * Populate the stream with events
+
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('c2309eec', 37.7877, -122.4205);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('18f4ea86', 37.3903, -122.0643);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4ab5cbad', 37.3952, -122.0813);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('8b6eae59', 37.3944, -122.0813);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4a7c7b41', 37.4049, -122.0822);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4ddad000', 37.7857, -122.4011);")
+
+# * Run a Pull query against the materialized view
+
+Ksql::Client.query('SELECT * from ridersNearMountainView WHERE distanceInMiles <= 10;')
+
+# * Clean up ksqlDB
+
+stream.close
+
+Ksql::Client.ksql('DROP TABLE IF EXISTS ridersNearMountainView;')
+Ksql::Client.ksql('DROP TABLE IF EXISTS currentLocation;')
+Ksql::Client.ksql('DROP STREAM IF EXISTS riderLocations;')

data/lib/generators/ksql.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+# Creates the Ksql initializer file for Rails apps.
+#
+# @example Invokation from terminal
+#   rails generate ksql
+#
+class KsqlGenerator < Rails::Generators::Base
+  desc "Description:\n  This creates a Rails initializer for Ksql"
+
+  source_root File.expand_path('templates', __dir__)
+
+  desc 'Configures Ksql to connect to ksqlDB'
+  def generate_layout
+    template 'initializer.rb', 'config/initializers/ksql.rb'
+  end
+end

data/lib/generators/templates/initializer.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+Ksql.configure do |config|
+  config.host = 'http://localhost:8088' # required
+  # config.auth = 'user:password' # optional
+end

data/lib/ksql/api/close_query.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Api
+    class CloseQuery
+      Headers = { 'Accept' => 'application/json' }.freeze
+
+      #
+      # Build the ksqlDB /close-query request
+      #
+      # @param [String] id Query ID
+      # @param [Hash] headers Request headers
+      #
+      # @return [Ksql::Connection::Request] Request instance
+      #
+      def self.build(id, headers:)
+        ::Ksql::Connection::Request.new(
+          { queryId: id },
+          '/close-query',
+          Headers.merge(headers),
+          :post
+        )
+      end
+    end
+  end
+end

data/lib/ksql/api/ksql.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Api
+    class Ksql
+      Headers = { 'Accept' => 'application/json' }.freeze
+
+      #
+      # Build the ksqlDB /ksql request
+      #
+      # @param [String] ksql SQL Statement
+      # @param [Integer] command_sequence_number The statements will not be run until all existing commands have completed.
+      # @param [Hash] headers Request headers
+      # @param [Hash] session_variables Variable substitution values
+      # @param [Hash] streams_properties Property overrides to run the statements with
+      #
+      # @return [Ksql::Connection::Request] Request instance
+      #
+      def self.build(ksql, command_sequence_number:, headers:, session_variables:, streams_properties:)
+        ::Ksql::Connection::Request.new(
+          {
+            ksql: ksql,
+            commandSequenceNumber: command_sequence_number,
+            sessionVariables: session_variables,
+            streamsProperties: streams_properties,
+          }.compact,
+          '/ksql',
+          Headers.merge(headers),
+          :post
+        )
+      end
+    end
+  end
+end

data/lib/ksql/api/query.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Api
+    class Query
+      Headers = { 'Accept' => 'application/json' }.freeze
+
+      #
+      # Build the ksqlDB /query-stream request
+      #
+      # @param [String] sql SQL Statement
+      # @param [Hash] headers Request headers
+      # @param [Hash] properties Optional properties for the query
+      # @param [Hash] session_variables Variable substitution values
+      #
+      # @return [Ksql::Connection::Request] Request instance
+      #
+      def self.build(sql, headers:, properties:, session_variables:)
+        ::Ksql::Connection::Request.new(
+          {
+            sql: sql,
+            properties: properties,
+            sessionVariables: session_variables
+          }.compact,
+          '/query-stream',
+          self::Headers.merge(headers),
+          :post
+        )
+      end
+    end
+  end
+end

data/lib/ksql/api/stream.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Api
+    class Stream < Query
+      Headers = { 'Accept' => 'application/vnd.ksqlapi.delimited.v1' }.freeze
+
+      # Inherits from Ksql::Api::Query overriding the request Accept header
+    end
+  end
+end

data/lib/ksql/client.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Ksql
+  class Client
+    class << self
+      #
+      # Request /close-query endpoint
+      #
+      # @param [String] id Query ID
+      # @param [Hash] headers Request headers
+      #
+      # @return [Array/Hash/String/Integer] Request result
+      #
+      def close_query(id, headers: {})
+        request = Api::CloseQuery.build(id, headers: headers)
+        result = Connection::Client.call_sync(request)
+        Handlers::Raw.handle(result)
+      end
+
+      #
+      # Request /ksql endpoint
+      #
+      # @param [String] ksql SQL Statement
+      # @param [Integer] command_sequence_number The statements will not be run until all existing commands have completed.
+      # @param [Hash] headers Request headers
+      # @param [Hash] session_variables Variable substitution values
+      # @param [Hash] streams_properties Property overrides to run the statements with
+      #
+      # @return [Ksql::@type] Request result
+      #
+      def ksql(ksql, command_sequence_number: nil, headers: {}, session_variables: {}, streams_properties: {})
+        request = Api::Ksql.build(ksql, command_sequence_number: command_sequence_number, headers: headers, session_variables: session_variables, streams_properties: streams_properties)
+        result = Connection::Client.call_sync(request)
+        Handlers::TypedRow.handle(result)
+      end
+
+      #
+      # Request /query-stream endpoint synchronously
+      #
+      # @param [String] sql SQL Statement
+      # @param [Hash] headers Request headers
+      # @param [Hash] properties Optional properties for the query
+      # @param [Hash] session_variables Variable substitution values
+      #
+      # @return [Ksql::Connection::Request] Request result
+      #
+      def query(sql, headers: {}, properties: {}, session_variables: {})
+        request = Api::Query.build(sql, headers: headers, properties: properties, session_variables: session_variables)
+        result = Connection::Client.call_sync(request)
+        Handlers::Collection.handle(result)
+      end
+
+      #
+      # Request /query-stream endpoint asynchronously
+      #
+      # @param [String] sql SQL Statement
+      # @param [Hash] headers Request headers
+      # @param [Hash] properties Optional properties for the query
+      # @param [Hash] session_variables Variable substitution values
+      #
+      # @return [Ksql::Stream] Stream instance
+      #
+      def stream(sql, headers: {}, properties: {}, session_variables: {})
+        request = Api::Stream.build(sql, headers: headers, properties: properties, session_variables: session_variables)
+        client, prepared_request = Connection::Client.call_async(request)
+        Handlers::Stream.handle(client, prepared_request)
+      end
+    end
+  end
+end

data/lib/ksql/collection.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Ksql
+  class Collection < Struct.new(:rows)
+    include Enumerable
+
+    #
+    # * Generate a class to fit ksqlDB query returned data
+    # * Populate the collection
+    #
+    # @param [Hash] struct_schema Struct Schema definition
+    # @param [Array] items Collection rows
+    #
+    def initialize(struct_schema, items)
+      struct = Ksql.const_set(id_to_struct(struct_schema['queryId']), Class.new(Struct.new(*struct_schema['columnNames'].map { |n| n.downcase.to_sym })))
+      self.rows = items.map { |i| struct.new(*i) }
+    end
+
+    #
+    # Allow iterations block on Rows
+    #
+    # @param [Block] &block Block to execute on each row
+    #
+    # @return [Array] Rows enumerable
+    #
+    def each(&block)
+      rows.each do |r|
+        yield(r)
+      end
+    end
+
+    private
+
+      #
+      # Dynamically generate a name based on the ksqlDB Query ID
+      #
+      # @param [String] id ksqlDB Query ID
+      #
+      # @return [String] Collection element Struct name
+      #
+      def id_to_struct(id)
+        struct_id = id.present? ? id.strip.gsub('-', '_') : SecureRandom.hex
+        "Query#{struct_id}Row"
+      end
+  end
+end

data/lib/ksql/configuration.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+class Ksql::Configuration < OpenStruct; end

data/lib/ksql/connection/client.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'net-http2'
+
+module Ksql
+  module Connection
+    class Client
+      class << self
+        #
+        # Execute the HTTP2 Sync Request
+        #
+        # @param [Ksql::Connection::Request] request Built request
+        #
+        # @return [Ksql::Connection::Response] HTTP2 Request response
+        #
+        def call_sync(request)
+          response = client.call(*request.to_params)
+          ::Ksql::Connection::Response.new(body: response.body, headers: response.headers)
+        end
+
+        #
+        # Prepare the HTTP2 Async Request based on the built input request
+        #
+        # @param [Ksql::Connection::Request] request Built request
+        #
+        # @return [Array] Client, Built Async Request
+        #
+        def call_async(request)
+          @@client = client
+          prepared_request = @@client.prepare_request(*request.to_params)
+          return @@client, prepared_request
+        end
+
+        private
+
+          #
+          # Return HTTP2 Client instance
+          #
+          # @return [NetHttp2::Client] HTTP2 Client
+          #
+          def client
+            NetHttp2::Client.new(::Ksql.config.host)
+          end
+      end
+    end
+  end
+end

data/lib/ksql/connection/request.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Connection
+    class Request < Struct.new(:body, :endpoint, :headers, :method)
+      #
+      # Returns the request params
+      #
+      # @return [Array] Request params
+      #
+      def to_params
+        return method, endpoint, { body: body.to_json, headers: headers.merge(auth_headers) }
+      end
+
+      private
+
+        #
+        # Prepares Authorization headers if Auth is configured
+        #
+        # @return [Hash] Authorization headers
+        #
+        def auth_headers
+          ::Ksql.config.auth.present? ? { 'Authorization' => "Basic #{::Ksql.config.auth}" } : {}
+        end
+    end
+  end
+end

data/lib/ksql/connection/response.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Connection
+    class Response
+      attr_reader :body, :headers
+
+      STATUS_KEY = ':status'.freeze
+
+      def initialize(body:, headers:)
+        @body = JSON.parse(body)
+        @headers = headers
+      end
+
+      #
+      # Check whether or not a Request has returned an Error
+      #
+      # @return [Boolean] True if error
+      #
+      def error?
+        !headers[STATUS_KEY].to_s.match?(/20[01]/)
+      end
+    end
+  end
+end

data/lib/ksql/error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Ksql
+  class Error < OpenStruct
+    # Description here
+  end
+end

data/lib/ksql/handlers/collection.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Handlers
+    class Collection
+      #
+      # Handle the response to generate an Enumerable collection
+      #
+      # @param [Ksql::Connection::Response] response HTTP2 Request response
+      #
+      # @return [Ksql::Collection] Collection result
+      #
+      def self.handle(response)
+        return Ksql::Error.new(response.body) if response.error?
+
+        body_dup = response.body
+        headers = body_dup.shift
+        Ksql::Collection.new(headers, body_dup)
+      end
+    end
+  end
+end

data/lib/ksql/handlers/raw.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Handlers
+    class Raw
+      #
+      # Return the Response raw parsed body
+      #
+      # @param [Ksql::Connection::Response] response HTTP2 Request response
+      #
+      # @return [Array/Hash/String/Integer] Response parsed body
+      #
+      def self.handle(response)
+        return Ksql::Error.new(response.body) if response.error?
+
+        response.body
+      end
+    end
+  end
+end

data/lib/ksql/handlers/stream.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Handlers
+    class Stream
+      #
+      # Instanciate a Ksql::Stream class to handle the streamed connection
+      #
+      # @param [Ksql::Connection::Client] client Client instance
+      # @param [Ksql::Connection::Request] request Built request
+      #
+      # @return [Ksql::Stream] Stream instance
+      #
+      def self.handle(client, request)
+        ::Ksql::Stream.new(client, request)
+      end
+    end
+  end
+end

data/lib/ksql/handlers/typed_row.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Ksql
+  module Handlers
+    class TypedRow
+      #
+      # Define and instanciate an OpenStruct to fit the typed request response
+      #
+      # @param [Ksql::Connection::Response] response HTTP2 Request response
+      #
+      # @return [Ksql::@type] OpenStruct instance
+      #
+      def self.handle(response)
+        return Ksql::Error.new(response.body) if response.error?
+        return response.body unless response.body.present?
+
+        parsed_body = response.body.first
+        row_type = parsed_body.delete('@type').camelize
+        row_class = Ksql.const_defined?(row_type) ? "Ksql::#{row_type}".constantize : Ksql.const_set(row_type, Class.new(OpenStruct))
+        row_class.new(parsed_body)
+      end
+    end
+  end
+end

data/lib/ksql/stream.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Ksql
+  class StreamError < StandardError; end
+
+  class Stream
+    attr_reader :id
+
+    def initialize(client, request)
+      @client = client
+      @request = request
+    end
+
+    #
+    # Close the streaming connection
+    #
+    def close
+      raise StreamError.new('The stream hasn\'t stared!') unless @id.present?
+
+      @client.close
+    end
+
+    #
+    # Specify the action to take when the Streaming connection gets closed.
+    #
+    # @param [Block] &block Code to execute on connection closure
+    #
+    def on_close(&block)
+      @client.on(:close) { yield }
+    end
+
+    #
+    # Specify the action to take when the Streaming connection raises an error.
+    #
+    # @param [Block] &block Code to execute when connection errors occur
+    #
+    def on_error(&block)
+      @client.on(:error) { |e| yield(e) }
+    end
+
+    #
+    # Streaming connection handler
+    #
+    # * Start the stream
+    # * Wrap the stream events into OpenStruct instances
+    # * Execute the passed block
+    #
+    # @param [Block] &block Code to execute each time an event arrives
+    #
+    def start(&block)
+      @headers = {}
+
+      @request.on(:headers) { |headers| @headers = headers }
+
+      @request.on(:body_chunk) do |body|
+        next unless body.present?
+
+        response = Ksql::Connection::Response.new(body: body, headers: @headers)
+        raise Ksql::StreamError.new(response.body['message']) if response.error?
+
+        if response.body.is_a? Hash
+          @event_class = build_event_class(response.body)
+          next
+        else
+          event = @event_class.new(*response.body)
+        end
+
+        yield(event)
+      end
+
+      @client.call_async(@request)
+    end
+
+    private
+
+      #
+      # Define a Struct class to fit the streaming data into.
+      #
+      # @param [Hash] schema Query schema
+      #
+      # @return [Ksql::Row] Stream event class
+      #
+      def build_event_class(schema)
+        @id = schema['queryId']
+        row_const = "Stream#{schema['queryId'].gsub('-', '_')}Row"
+        Ksql.const_defined?(row_const) ? "Ksql::#{row_const}".constantize : Ksql.const_set(row_const, Class.new(Struct.new(*schema['columnNames'].map { |c| c.downcase.to_sym })))
+      end
+  end
+end

data/lib/ksql/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ksql
+  VERSION = '0.1.0.beta'
+end

data/lib/ksql.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/string'
+require 'json'
+require 'ostruct'
+
+require_relative 'ksql/api/close_query'
+require_relative 'ksql/api/ksql'
+require_relative 'ksql/api/query'
+require_relative 'ksql/api/stream'
+
+require_relative 'ksql/handlers/collection'
+require_relative 'ksql/handlers/raw'
+require_relative 'ksql/handlers/stream'
+require_relative 'ksql/handlers/typed_row'
+
+require_relative 'ksql/connection/client'
+require_relative 'ksql/connection/request'
+require_relative 'ksql/connection/response'
+
+require_relative 'ksql/client'
+require_relative 'ksql/collection'
+require_relative 'ksql/configuration'
+require_relative 'ksql/error'
+require_relative 'ksql/stream'
+
+require_relative 'ksql/version'
+
+module Ksql
+  def self.config
+    @config ||= Ksql::Configuration.new
+  end
+
+  def self.configure
+    yield(config)
+  end
+end

data/sig/ksql.rbs

@@ -0,0 +1,4 @@
+module Ksql
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end