ksql 0.1.0.beta

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2cd9a1d71cc892172f660cee8406014672b19bc7a510cf6897194201ab9fbf76
+  data.tar.gz: acef9c5aca159a91667816ef3b6c53c54bfa432e141bdff755cbad7ea7a3e986
+SHA512:
+  metadata.gz: 603741e0d00a8bc9c8193473600b2314c9cebb4fbbf0de321127ce52d91d78cfe70710a45d6ec1721e80109811645f207186b2c3aedab967c5485771e822bac6
+  data.tar.gz: 3e50792e2df0bca4ccf27fb8991ff727099d761284bff71303390fdaa23233ea12756dc92fda170d72ddcdfb4bb3112360c88658ca90a8a4f8f97ddbda58b5ea

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,175 @@
+# Allow longer lines and methods
+Layout/LineLength:
+  Max: 240
+
+Metrics/MethodLength:
+  Max: 30
+
+# Tests are declarative, no block length test
+Metrics/BlockLength:
+  IgnoredMethods: ['describe', 'context', 'namespace']
+
+AllCops:
+  TargetRubyVersion: 2.5
+  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
+  # to ignore them, so only the ones explicitly set in this file are enabled.
+  DisabledByDefault: true
+  Exclude:
+
+# Prefer &&/|| over and/or.
+Style/AndOr:
+  Enabled: true
+
+# Align `when` with `case`.
+Layout/CaseIndentation:
+  Enabled: true
+
+# Align comments with method definitions.
+Layout/CommentIndentation:
+  Enabled: true
+
+Layout/ElseAlignment:
+  Enabled: true
+
+# Align `end` with the matching keyword or starting expression except for
+# assignments, where it should be aligned with the LHS.
+Layout/EndAlignment:
+  Enabled: true
+  EnforcedStyleAlignWith: variable
+  AutoCorrect: true
+
+Layout/EmptyLineAfterMagicComment:
+  Enabled: true
+
+# In a regular class definition, no empty lines around the body.
+Layout/EmptyLinesAroundClassBody:
+  Enabled: true
+
+# In a regular method definition, no empty lines around the body.
+Layout/EmptyLinesAroundMethodBody:
+  Enabled: true
+
+# In a regular module definition, no empty lines around the body.
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: true
+
+Layout/FirstArgumentIndentation:
+  Enabled: true
+
+# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
+Style/HashSyntax:
+  Enabled: true
+
+# Method definitions after `private` or `protected` isolated calls need one
+# extra level of indentation.
+Layout/IndentationConsistency:
+  Enabled: true
+  EnforcedStyle: indented_internal_methods
+
+# Two spaces, no tabs (for indentation).
+Layout/IndentationWidth:
+  Enabled: true
+
+Layout/LeadingCommentSpace:
+  Enabled: true
+
+Layout/SpaceAfterColon:
+  Enabled: true
+
+Layout/SpaceAfterComma:
+  Enabled: true
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  Enabled: true
+
+Layout/SpaceAroundKeyword:
+  Enabled: true
+
+Layout/SpaceAroundOperators:
+  Enabled: true
+
+Layout/SpaceBeforeComma:
+    Enabled: true
+
+Layout/SpaceBeforeFirstArg:
+    Enabled: true
+
+Style/DefWithParentheses:
+  Enabled: true
+
+# Defining a method with parameters needs parentheses.
+Style/MethodDefParentheses:
+  Enabled: true
+
+# Style/FrozenStringLiteralComment:
+#   Enabled: true
+#   EnforcedStyle: always
+#   Exclude:
+#     - 'actionview/test/**/*.builder'
+#     - 'actionview/test/**/*.ruby'
+#     - 'actionpack/test/**/*.builder'
+#     - 'actionpack/test/**/*.ruby'
+#     - 'activestorage/db/migrate/**/*.rb'
+#     - 'db/migrate/**/*.rb'
+#     - 'db/*.rb'
+
+# Use `foo {}` not `foo{}`.
+Layout/SpaceBeforeBlockBraces:
+  Enabled: true
+
+# Use `foo { bar }` not `foo {bar}`.
+Layout/SpaceInsideBlockBraces:
+  Enabled: true
+
+# Use `{ a: 1 }` not `{a:1}`.
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+Layout/SpaceInsideParens:
+  Enabled: true
+
+# Check quotes usage according to lint rule below.
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: single_quotes
+
+# Detect hard tabs, no hard tabs.
+Layout/IndentationStyle:
+  Enabled: true
+
+# Blank lines should not have any spaces.
+Layout/TrailingEmptyLines:
+  Enabled: true
+
+# No trailing whitespace.
+Layout/TrailingWhitespace:
+  Enabled: true
+
+# Use quotes for string literals when they are enough.
+Style/RedundantPercentQ:
+  Enabled: true
+
+# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
+Lint/RequireParentheses:
+  Enabled: true
+
+Lint/RedundantStringCoercion:
+  Enabled: true
+
+# Supports --auto-correct
+Style/RedundantSelf:
+  Description: Don't use self where it's not needed.
+  StyleGuide: https://github.com/rubocop-hq/ruby-style-guide#no-self-unless-required
+  Enabled: true
+
+Style/RedundantReturn:
+  Enabled: true
+  AllowMultipleReturnValues: true
+
+Style/Semicolon:
+  Enabled: true
+  AllowAsExpressionSeparator: true
+
+# Prefer Foo.method over Foo::method
+Style/ColonMethodCall:
+  Enabled: true

data/CHANGELOG.md

@@ -0,0 +1,23 @@
+## [Releases]
+
+## [0.1.0.beta] - 2022-02-24
+
+- Beta release
+
+### Added
+
+- Authentication header in config
+- `/close-query` endpoint support
+- `/ksql` endpoint support
+- `/query-stream` endpoint support
+
+## [0.1.0] - 2022-02-21 (Yanked)
+
+- First release
+
+### Added
+
+- Statements
+- Push Queries
+- Pull Queries
+- Close Queries

data/Gemfile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ksql.gemspec
+gemspec
+
+gem 'byebug', '~> 11'
+
+gem 'rake', '~> 13'
+
+gem 'rspec', '~> 3'
+
+gem 'rubocop', '~> 1'

data/Gemfile.lock

@@ -0,0 +1,88 @@
+PATH
+  remote: .
+  specs:
+    ksql (0.1.0)
+      activesupport (>= 5)
+      hashie (~> 4)
+      net-http2 (~> 0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    crack (0.4.5)
+      rexml
+    diff-lcs (1.5.0)
+    hashdiff (1.0.1)
+    hashie (4.1.0)
+    http-2 (0.11.0)
+    i18n (1.9.1)
+      concurrent-ruby (~> 1.0)
+    minitest (5.15.0)
+    net-http2 (0.18.4)
+      http-2 (~> 0.11)
+    parallel (1.21.0)
+    parser (3.0.3.2)
+      ast (~> 2.4.1)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.2)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    rubocop (1.24.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.15.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.15.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    vcr (6.0.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  byebug (~> 11)
+  ksql!
+  rake (~> 13)
+  rspec (~> 3)
+  rubocop (~> 1)
+  vcr (~> 6)
+  webmock (~> 3)
+
+BUNDLED WITH
+   2.3.7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Lapo Elisacci
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,319 @@
+![][ruby-shield]
+
+# ![ksql-rocket](https://user-images.githubusercontent.com/50866745/155471557-6fe01746-444f-4ff6-a164-eeeb3b4b53a1.png) Ruby ksqlDB Client (Under QA)
+
+![](https://img.shields.io/static/v1?label=Latest&message=0.1.0&color=blue)
+![](https://img.shields.io/static/v1?label=Coverage&message=95%&color=green)
+![](https://img.shields.io/static/v1?label=Documentation&message=98%&color=success)
+![](https://img.shields.io/static/v1?label=Mantained?&message=Yes&color=success)
+
+KSQL is a [ksqlDB](https://ksqldb.io/) Ruby client that focuses on ease of use. Supports all recent ksqlDB features and does not have any heavyweight dependencies.
+
+## What is ksqlDB?
+
+ksqlDB is a database purpose-built for [Apache Kafka®](https://kafka.apache.org/) streams processing applications, more details [here](https://ksqldb.io/).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ksql'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install ksql
+
+## Usage
+
+The gem allows you to perform requests to ksqlDB REST API.
+Checkout the ksqlDB official documentation [here](https://docs.ksqldb.io/en/latest/developer-guide/api/).
+
+### Configuration
+
+The gem requires a minimum configuration to connect to ksqlDB, it is shipped with a built-in generator to create a Rails initializer.
+
+
+    $ rails generate ksql
+    
+```Ruby
+Ksql.configure do |config|
+  config.host = 'http://localhost:8088' # Required
+  # config.auth = 'user:password' # optional
+end
+```
+
+**The Client supports Basic Authentication only**
+
+### [Statements](https://docs.ksqldb.io/en/latest/developer-guide/ksqldb-rest-api/ksql-endpoint/)
+
+All statements, except those starting with `SELECT` can be run with the `ksql` method:
+
+```Ruby
+Ksql::Client.ksql("SHOW TABLES;")
+
+# Or
+
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('18f4ea86', 37.3903, -122.0643);")
+```
+
+You can also pass optional `properties` to the method (Check the official documentation if needed).
+
+```Ruby
+Ksql::Client.ksql("INSERT INTO ...", command_sequence_number: 42, streams_properties: { ... }, session_variables: { ... })
+```
+
+As well as custom `Headers`.
+
+```Ruby
+Ksql::Client.ksql("INSERT INTO ...", headers: { ... })
+```
+
+## [Queries](https://docs.ksqldb.io/en/latest/concepts/queries)
+
+ksqlDB has three kinds of Queries:
+
+### [Persistent Query](https://docs.ksqldb.io/en/latest/concepts/queries/#persistent)
+
+`Persistent queries` are server-side queries and there's nothing much to say here.
+
+In case you want to close a `Persistent` Query you can do so with the `close_query` method, passing the Query ID:
+
+```Ruby
+Ksql::Client.close_query("CTAS_RIDERSNEARMOUNTAINVIEW_5")
+```
+
+**WARNING:** There's a known issue here, read [below](#ksqldb-close-query).
+
+### [Push Query](https://docs.ksqldb.io/en/latest/concepts/queries/#push)
+
+`Push` queries enable you to query a stream or materialized table with a subscription to the results. 
+They run asynchronously and you can spot them as they include the `EMIT` keyword at the end of the SQL statement.
+
+To define a `Push` query connection simply call the `stream` method:
+
+```Ruby
+stream = Ksql::Client.stream("SELECT * FROM riderLocations EMIT CHANGES;")
+```
+
+You can specify what action to take when an exception gets raised:
+
+```Ruby
+stream.on_error do |e|
+  puts e.message
+end
+```
+
+or what to do when the connection gets closed:
+
+```Ruby
+stream.on_close do
+  puts 'Session closed!'
+end
+```
+
+To start the stream, call the `start` method, It accepts a block to execute each time a message gets recieved:
+
+```Ruby
+stream.start do |location|
+  # The streaming events get wrapped inside an ORM-like Class
+  puts location.latitude
+  puts location.longitude
+end
+```
+
+**CAREFUL:** The block gets executed inside a separated Thread, make sure your code is thread safe!
+
+You can close the connection by calling the `close` method:
+
+```Ruby
+stream.close
+```
+
+Example:
+
+```Ruby
+# Define the Stream
+stream = Ksql::Client.stream("SELECT * FROM riderLocations EMIT CHANGES;")
+
+# Start the connection
+stream.start do |location|
+  # This code will get executed inside a separated Thread
+  puts location.latitude
+  puts location.longitude
+end
+
+# Do something
+sleep(10)
+
+# Close the connection
+stream.close
+```
+
+You can also specify custom `Properties` as well as `Headers` and `Session Variables`:
+
+```Ruby
+stream = Ksql::Client.stream("SELECT * FROM riderLocations EMIT CHANGES;", headers: { ... }, properties: { ... }, session_variables: { ... })
+# ...
+```
+
+### [Pull Query](https://docs.ksqldb.io/en/latest/concepts/queries/#push)
+
+`Pull` queries are the most "traditional" ones, they run synchronously and they can be executed with the `query` method"
+
+```Ruby
+locations = Ksql::Client.query("SELECT * FROM riderLocations;")
+```
+
+An Enumerable collection of ORM-like Objects is returned.
+Iteration methods are available on the collection:
+
+```Ruby
+locations.each do |location|
+  # do something
+end
+
+locations.map do |location|
+  # do something
+end
+
+locations.count
+```
+
+## Example
+
+The following example is from the official ksqlDB [Quickstart](https://ksqldb.io/quickstart.html):
+
+```Ruby
+require 'ksql'
+require 'logger'
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+
+# Create a stream
+
+Ksql::Client.ksql("CREATE STREAM riderLocations (profileId VARCHAR, latitude DOUBLE, longitude DOUBLE)
+  WITH (kafka_topic='locations', value_format='json', partitions=1);")
+
+# Create materialized views
+
+Ksql::Client.ksql("CREATE TABLE currentLocation AS
+  SELECT profileId,
+         LATEST_BY_OFFSET(latitude) AS la,
+         LATEST_BY_OFFSET(longitude) AS lo
+  FROM riderlocations
+  GROUP BY profileId
+  EMIT CHANGES;")
+
+# Create materialized views
+
+Ksql::Client.ksql("CREATE TABLE ridersNearMountainView AS
+  SELECT ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1) AS distanceInMiles,
+         COLLECT_LIST(profileId) AS riders,
+         COUNT(*) AS count
+  FROM currentLocation
+  GROUP BY ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1);")
+
+# Run a push query over the stream
+
+stream = Ksql::Client.stream("SELECT * FROM riderLocations
+  WHERE GEO_DISTANCE(latitude, longitude, 37.4133, -122.1162) <= 5 EMIT CHANGES;")
+
+# Handle exceptions
+
+stream.on_error do |e|
+  logger.error(e.message)
+end
+
+# Handle stream close
+
+stream.on_close do
+  logger.info("Stream closed!")
+end
+
+# Start the stream
+
+stream.start do |location|
+  # Print the result into a file
+  File.open('output.log', 'a') do |f|
+    f.puts "Latitude: #{location.latitude}, Longitude: #{location.longitude}"
+  end
+end
+
+# Populate the stream with events
+
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('c2309eec', 37.7877, -122.4205);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('18f4ea86', 37.3903, -122.0643);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('4ab5cbad', 37.3952, -122.0813);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('8b6eae59', 37.3944, -122.0813);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('4a7c7b41', 37.4049, -122.0822);")
+Ksql::Client.ksql("INSERT INTO riderLocations (profileId, latitude, longitude)
+  VALUES ('4ddad000', 37.7857, -122.4011);")
+
+# Run a Pull query against the materialized view
+
+locations = Ksql::Client.query("SELECT * from ridersNearMountainView
+  WHERE distanceInMiles <= 10;")
+
+# Close the stream
+
+stream.close
+
+# Drop
+
+Ksql::Client.ksql('DROP TABLE IF EXISTS ridersNearMountainView;')
+Ksql::Client.ksql('DROP TABLE IF EXISTS currentLocation;')
+Ksql::Client.ksql('DROP STREAM IF EXISTS riderLocations;')
+
+```
+
+## Supported ksqlDB versions
+
+| Version |  |
+| ------ | -- |
+| 0.23.1 | :heavy_check_mark: Supported |
+| 0.22.0 | :heavy_check_mark: Supported |
+| 0.21.0 | :heavy_check_mark: Supported |
+| 0.20.0 | :heavy_check_mark: Supported |
+| 0.19.0 | :heavy_check_mark: Supported |
+| 0.18.0 | :heavy_check_mark: Supported |
+| 0.17.0 | :x: Untested |
+| Older | :x: Untested |
+
+## Known issues
+
+### ksqlDB close-query
+
+Although it actually works, at the moment, the latest release of ksqlDB returns an error each time you request the `/close-query` endpoint.
+Therefore the query will correctly get closed but an error will get returned anyways.
+
+Official issue [here](https://github.com/confluentinc/ksql/issues/8251).
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+Make sure you have a working instance of ksqlDB, you can find the official quickstart [here](https://ksqldb.io/quickstart.html).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/LapoElisacci/ksql.
+
+## License
+
+[![Licence](https://img.shields.io/github/license/Ileriayo/markdown-badges?style=for-the-badge)](https://opensource.org/licenses/MIT)
+
+<!--- MARKDOWN LINKS --->
+
+[ruby-shield]: https://img.shields.io/badge/Ruby-CC342D?style=for-the-badge&logo=ruby&logoColor=white

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]