qravan 0.1.0

data/ext/qravan/qravan.c

@@ -0,0 +1,9 @@
+#include "qravan.h"
+
+VALUE rb_mQravan;
+
+void
+Init_qravan(void)
+{
+  rb_mQravan = rb_define_module("Qravan");
+}

data/ext/qravan/qravan.h

@@ -0,0 +1,6 @@
+#ifndef QRAVAN_H
+#define QRAVAN_H 1
+
+#include "ruby.h"
+
+#endif /* QRAVAN_H */

data/falcon.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env -S falcon host
+# frozen_string_literal: true
+
+load :rack, :lets_encrypt_tls, :supervisor
+
+hostname = File.basename(__dir__)
+rack hostname, :lets_encrypt_tls do
+  cache true
+  endpoint Async::HTTP::Endpoint.parse("http://localhost:3300").with(protocol: Async::HTTP::Protocol::HTTP1)
+end
+
+supervisor

data/lib/qravan/async_loader.rb

@@ -0,0 +1,33 @@
+require "async"
+require "async/barrier"
+require "async/http/internet"
+
+class AsyncLoader
+  def self.perform(urls)
+    internet = Async::HTTP::Internet.new
+    barrier = Async::Barrier.new
+    time = Time.now
+    requests = 0
+    result = []
+    urls.each do |url|
+      barrier.async do
+        Console.logger.info "AsyncHttp#get: #{url}"
+        begin
+          response = internet.get(url)
+          body = JSON.parse(response.read)
+          result << { url: url, body: body }
+          requests += 1
+        ensure
+          response.finish
+        end
+        Console.logger.info "AsyncHttp#fulfill: #{url}"
+      end
+    end
+
+    Console.logger.info "AsyncHttp#wait"
+    barrier.wait
+    total = Time.now - time
+    result.unshift({time: "Duration: #{Time.now - time}s for #{requests} (RPS: #{(requests / total).to_i} r/s)"})
+    result.to_json
+  end
+end

data/lib/qravan/cargo.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Qravan
+  # Load models and sources to cargo
+  class Cargo
+    attr_accessor :sources, :models, :connections
+
+    def initialize
+      Console.logger.info "Cargo initialized:"
+      Console.logger.info "Sources: #{sources.count}"
+      Console.logger.info "Models: #{models.count}"
+      Console.logger.info "Connections: #{connections.count}"
+    end
+
+    def sources
+      @sources ||= preload_sources
+    end
+
+    def models
+      @models ||= preload_models
+    end
+
+    def connections
+      @connections ||= connect_sources
+    end
+
+    def preload_sources
+      @sources = {}
+      list("sources").each do |source|
+        Console.logger.info "Reading source #{source}"
+        preloaded_source = YAML.load_file("#{source}/1.0/source.yaml")
+        @sources[preloaded_source.keys.first] = preloaded_source.first.last
+      end
+      @sources
+    end
+
+    def concat_models
+      model_files = {}
+      (list("models") - %w[models/current models/base]).each do |model|
+        Console.logger.info "Reading model file #{model}/1.0/model.yaml"
+        model_files[model] = File.readlines("#{model}/1.0/model.yaml").map(&:chomp)
+      end
+
+      File.open("models/current/model.yaml", "w") do |output_file|
+        output_file.puts File.readlines("models/base/1.0/model.yaml").map(&:chomp)
+        model_files.each { |elem| output_file.puts elem[1] }
+      end
+    end
+
+    def preload_models
+      concat_models
+      @models = {}
+      Console.logger.info "Preparing models"
+      preloaded_models = YAML.load_file("models/current/model.yaml")
+      preloaded_models["resources"].each do |model|
+        @models[model.keys.first] = model
+      end
+      @models
+    end
+
+    def connect_sources
+      @connections = {}
+      sources.each do |source|
+        case source[1]["type"]
+        when "rest"
+          @connections[source[0]] = Qravan::Rest.new(source[1])
+        when "db"
+          @connections[source[0]] = Qravan::Db.new(source[1])
+        else
+          Console.logger.warn "Not supported resource"
+        end
+      end
+      @connections
+    end
+
+    def list(what = "sources")
+      Dir["#{what}/*"]
+    end
+  end
+end

data/lib/qravan/credentials/request.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Qravan
+  module Credentials
+    # Request credentials
+    class Request
+      attr_accessor :credentials
+
+      def initialize(credentials)
+        @credentials = credentials["request"]
+      end
+
+      def prepare
+        {
+          id: id,
+          sub_id: sub_id,
+          name: name,
+          purpose_id: purpose,
+          audit: audit,
+          audit_id: audit_id,
+          audit_token: audit_token
+        }
+      end
+
+      def id
+        @credentials["id"]
+      end
+
+      def sub_id
+        @credentials["sub_id"]
+      end
+
+      def name
+        @credentials["name"]
+      end
+
+      def purpose
+        @credentials["purpose_id"]
+      end
+
+      def audit
+        @credentials["audit"]
+      end
+
+      def audit_id
+        @credentials["audit_id"]
+      end
+
+      def audit_token
+        @credentials["audit_token"]
+      end
+    end
+  end
+end

data/lib/qravan/credentials/response.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Qravan
+  module Credentials
+    # Response credentials
+    class Response
+      attr_accessor :credentials
+
+      def initialize(response_credentials = {})
+        @credentials = response_credentials
+      end
+
+      def prepare
+        {
+          id: id,
+          sub_id: sub_id,
+          started_at: started_at,
+          finished_at: finished_at
+        }
+      end
+
+      def id
+        SecureRandom.uuid
+      end
+
+      def sub_id
+        SecureRandom.uuid
+      end
+
+      def started_at
+        Time.now
+      end
+
+      def finished_at
+        Time.now
+      end
+    end
+  end
+end

data/lib/qravan/credentials/signature.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Qravan
+  module Credentials
+    # System credentials
+    class Signature
+      attr_accessor :credentials
+
+      def initialize(credentials)
+        @credentials = credentials["signature"]
+      end
+
+      def prepare
+        {
+          digest: digest,
+          signature: signature
+        }
+      end
+
+      def digest
+        @credentials["digest"]
+      end
+
+      def signature
+        @credentials["signature"]
+      end
+
+    end
+  end
+end

data/lib/qravan/credentials/system.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Qravan
+  module Credentials
+    # System credentials
+    class System
+      attr_accessor :credentials
+
+      def initialize(credentials)
+        @credentials = credentials["system"]
+      end
+
+      def prepare
+        {
+          mnemonic: mnemonic,
+          instance_id: instance,
+          user_id: user
+        }
+      end
+
+      def mnemonic
+        @credentials["mnemonic"]
+      end
+
+      def instance
+        @credentials["instance_id"]
+      end
+
+      def user
+        @credentials["user_id"]
+      end
+    end
+  end
+end

data/lib/qravan/credentials.rb

@@ -0,0 +1,29 @@
+require_relative "credentials/system"
+require_relative "credentials/request"
+require_relative "credentials/response"
+require_relative "credentials/signature"
+
+module Qravan
+  BANNER = %(
+        Data API Qravan Server
+        Developed at CAPAA by Alexander Panasenkov, 2022
+).freeze
+  module Credentials
+    class Query
+      attr_accessor :query_credentials
+
+      def initialize(query_credentials)
+        @query_credentials ||= query_credentials
+      end
+
+      def credentials
+        {
+          response: Qravan::Credentials::Response.new(@query_credentials).prepare,
+          system: Qravan::Credentials::System.new(@query_credentials).prepare,
+          request: Qravan::Credentials::Request.new(@query_credentials).prepare,
+          signature: Qravan::Credentials::Signature.new(@query_credentials).prepare
+        }
+      end
+    end
+  end
+end

data/lib/qravan/model.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Qravan
+  # Models for data extractions class
+  class Model
+    attr_accessor :models
+
+    def initialize(cargo = {})
+      @models ||= cargo.models
+    end
+
+    def call(env)
+      body = [models.to_json]
+      status  = 200
+      headers = { "content-type" => "application/json" }
+
+      [status, headers, body]
+    end
+  end
+end

data/lib/qravan/query/request.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require "async"
+require "async/barrier"
+
+module Qravan
+  # Requests processing class
+  class Request
+    attr_accessor :resources, :sources, :cargo
+
+    def initialize(request, cargo = {})
+      @query = request["query"]
+      @credentials = request["credentials"]
+      @resources = []
+      @cargo ||= cargo
+
+      validate
+      extract_resources
+    end
+
+    def validate
+      @query
+    end
+
+    def extract_resources
+      barrier = Async::Barrier.new
+      barrier.async do
+        @query.each do |key, resource|
+          @resources << { key => Qravan::Resource.new(key, resource, cargo).extract }
+        end
+      ensure
+        @resources
+      end
+      barrier.wait
+    end
+
+    def credentials
+      @credentials
+    end
+  end
+end

data/lib/qravan/query/response.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Qravan
+  class Response
+    attr_accessor :request
+    attr_accessor :resources
+    attr_accessor :credentials
+
+    def initialize(request)
+      @request ||= request
+      @resources = request.resources
+      @credentials = request.credentials
+    end
+
+    def validate
+      @request
+    end
+
+    def answer
+      {
+        "response": @resources,
+        "credentials": Qravan::Credentials::Query.new(@credentials).credentials
+      }.to_json
+    end
+  end
+end

data/lib/qravan/query.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+require_relative "query/request"
+require_relative "query/response"
+require_relative "resource/resource"
+require_relative "source/db"
+require_relative "source/rest"
+
+module Qravan
+
+  # Sources for data extractions class
+  class Query
+    attr_accessor :cargo
+
+    def initialize(cargo)
+      @cargo ||= cargo
+    end
+
+    def call(env)
+      query = Rack::Request.new(env)
+      request = Qravan::Request.new(JSON.parse(query.body.read), cargo)
+      response = Qravan::Response.new(request)
+
+      body = [response.answer]
+      status  = 200
+      headers = { "content-type" => "application/json" }
+
+      [status, headers, body]
+    end
+  end
+end

data/lib/qravan/resource/resource.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Qravan
+  # Resource processing class
+  class Resource
+    attr_accessor :connections, :cargo
+
+    def initialize(name, data, cargo = {})
+      @resource = name
+      @resource_data = data
+      @cargo ||= cargo
+      @connections = cargo.connections
+    end
+
+    def validate
+      @resource_data
+    end
+
+    def extract
+      source_name = "#{cargo.models[@resource]["extract"]["source"]["name"]}_source"
+      @connections[source_name.to_s].extract(sql)
+    end
+
+    def sql
+      [
+        :select,
+        @resource_data["attributes"].join(","),
+        :from,
+        cargo.models[@resource]["extract"]["source"]["table"],
+        :where,
+        @resource_data["conditions"].map { |k, v| "#{k}='#{v}'" }
+                                    .join(" and ")
+      ].join(" ")
+    end
+  end
+end

data/lib/qravan/source/db.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "async"
+require "sequel"
+require "logger"
+module Qravan
+  # Database class
+  class Db
+    attr_accessor :result, :connection
+
+    def initialize(source)
+      @connection = Sequel.connect(adapter: source["adapter"],
+                                   user: source["user"],
+                                   password: source["password"],
+                                   host: source["host"],
+                                   port: source["port"],
+                                   database: source["database"],
+                                   max_connections: source["max_connections"],
+                                   logger: Logger.new("logs/db.log"))
+
+    end
+
+    def extract(sql)
+      Console.logger.info "DB EXTRACT: #{sql}"
+      @result = []
+      time = Time.now
+      @connection[sql].each do |row|
+        @result << row
+      end
+      total = Time.now - time
+      @result << { time: "DB Duration: #{Time.now - time}s" }
+      @result
+    end
+  end
+end

data/lib/qravan/source/rest.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/barrier"
+require "logger"
+module Qravan
+  # Database class
+  class Rest
+    attr_accessor :result, :connection
+
+    def initialize(source)
+      @connection = source
+    end
+
+    def extract(sql = "select * from smevql1.driverlicenseql3;")
+      Console.logger.info "REST EXTRACT: #{sql}"
+      internet = Async::HTTP::Internet.new
+      barrier = Async::Barrier.new
+      time = Time.now
+      @result = []
+      url = "#{@connection["protocol"]}://#{@connection["host"]}:#{@connection["port"]}/#{@connection["path"]}"
+      headers = [%w[accept application/json], %w[content-type application/json]]
+      body = [@connection["template"] % { request: sql }]
+      barrier.async do
+        response = internet.post(url, headers, body)
+        response_body = JSON.parse(response.read)
+        @result << response_body[@connection["payload-path"]]
+      ensure
+        response.finish
+      end
+      barrier.wait
+      @result << { time: "REST Duration: #{Time.now - time}s " }
+      @result
+    end
+  end
+end

data/lib/qravan/source.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Qravan
+  # Sources for data extractions class
+  class Source
+    attr_accessor :sources
+
+    def initialize(cargo = {})
+      @sources ||= cargo.sources
+    end
+
+    def call(env)
+      body = [unpassworded.to_json]
+      status  = 200
+      headers = { "content-type" => "application/json" }
+
+      [status, headers, body]
+    end
+
+    def unpassworded
+      sources.map { |key, value| sources[key]["password"] = "******" if sources[key]["password"] }
+      sources
+    end
+  end
+end

data/lib/qravan/version.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Qravan
+  VERSION = "0.1.0"
+  ENV = "development"
+
+  class Spec
+
+    def call(env)
+      body = [Qravan::Spec.collect_spec.to_json]
+      status  = 200
+      headers = { "content-type" => "application/json" }
+
+      [status, headers, body]
+    end
+
+    class << self
+      def collect_spec
+        {
+          "spec": {
+            "server": {
+              "type": "Qravan Server",
+              "version": Qravan::VERSION,
+              "env": Qravan::ENV
+            },
+            "protocol": {
+              "type": "Qravan Spec",
+              "version": "1.0"
+            }
+          }
+        }
+      end
+    end
+  end
+end

data/lib/qravan.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "qravan/version"
+require_relative "qravan/cargo"
+require_relative "qravan/source"
+require_relative "qravan/model"
+require_relative "qravan/query"
+require_relative "qravan/credentials"
+
+require "async"
+require "async/http/internet"
+require "redis"
+
+module Qravan
+  class Error < StandardError; end
+
+  def self.root
+    File.dirname __dir__
+  end
+
+  def self.logs
+    File.join root, "logs"
+  end
+
+end