any_query 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4acc0ba4d4783462b9c1d7b985395b44ff91f857c519252379d0a9f4f2bbcc63
+  data.tar.gz: 71b6ed3cbceef45d8a4f696075faa680249e28b995bf3ea1406416c7b0be6334
+SHA512:
+  metadata.gz: 519287d516d95e87749cab7de90425622bab52466f5bc08de5c721bdab167222299452da647bb9d6e746ed24f28c1e3f5b3e07470f579326aa6db68450ec8e7f
+  data.tar.gz: bfadab83910a87cb40c0948ff5e71a7a4ce634b561e0295ffacaf5cc8919664d4305fdea553f1e724128cf6773b06506b0ca353b8caa902fb12e34a67215b5fd

data/lib/any_query/adapters/base.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  module Adapters
+    # @api private
+    class Base
+      def initialize(config)
+        @config = config.to_h
+      end
+
+      def load(model, select:, joins:, where:, limit:)
+        raise NotImplementedError
+      end
+
+      def load_single(model, id, joins)
+        load(model, select: [], joins:, where: [{ id: }], limit: 1).first
+      end
+
+      def instantiate_model(model, record)
+        instance = model.new
+        attrs = instance.instance_variable_get(:@attributes)
+        record.each do |key, value|
+          attrs.send("#{key}=", value)
+        end
+        instance
+      end
+
+      def resolve_path(data, path)
+        if path.is_a?(Proc)
+          path.call(data)
+        elsif path.is_a?(Array)
+          data.dig(*path)
+        else
+          data[path]
+        end
+      rescue StandardError => e
+        AnyQuery::Config.logger.error "Failed to resolve path #{path} on #{data.inspect}"
+        raise e
+      end
+
+      def fallback_where(data, wheres)
+        data.filter do |row|
+          wheres.all? do |where|
+            where.all? do |key, value|
+              resolve_path(row, key) == value
+            end
+          end
+        end
+      end
+
+      def resolve_join(data, join)
+        AnyQuery::Config.logger.debug "Joining #{join[:model]} on #{join[:primary_key]} = #{join[:foreign_key]}"
+        foreign_keys = data.map { |row| resolve_path(row, join[:foreign_key]) }.compact.uniq
+        result = run_external_join(join, foreign_keys)
+
+        result = group_join_data(result, join)
+
+        data.each do |row|
+          row[join[:into]] = result[resolve_path(row, join[:foreign_key])] || (join[:as] == :list ? [] : nil)
+        end
+      end
+
+      def group_join_data(data, join)
+        if join[:as] == :list
+          data.group_by { |e| resolve_path(e, join[:primary_key]) }
+        else
+          data.index_by { |e| resolve_path(e, join[:primary_key]) }
+        end
+      end
+
+      def run_external_join(join, foreign_keys)
+        case join[:strategy]
+        when Proc
+          join[:strategy].call(foreign_keys)
+        when :single
+          map_multi_threaded(foreign_keys.uniq) { |key| join[:model].find(key) }
+        when :full_scan
+          join[:model].to_a
+        else
+          join[:model].where(id: foreign_keys).to_a
+        end
+      end
+
+      def resolve_select(chain, select)
+        chain.map do |record|
+          select.map do |field|
+            resolve_path(record, field)
+          end
+        end
+      end
+
+      def map_multi_threaded(list, concurrency = 50)
+        list.each_slice(concurrency).flat_map do |slice|
+          slice
+            .map { |data| Thread.new { yield(data) } }
+            .each(&:join)
+            .map(&:value)
+        end
+      end
+
+      def parse_field_type(field, line)
+        method_name = "parse_field_type_#{field[:type]}"
+
+        if respond_to?(method_name)
+          send(method_name, field, line)
+        else
+          line.strip
+        end
+      rescue StandardError => e
+        AnyQuery::Config.logger.error "Failed to parse field \"#{line}\" with type #{field.inspect}: #{e.message}"
+        nil
+      end
+
+      def parse_field_type_integer(_, line)
+        line.to_i
+      end
+
+      def parse_field_type_date(field, line)
+        if field[:format]
+          Date.strptime(line.strip, field[:format])
+        else
+          Date.parse(line)
+        end
+      end
+
+      def parse_field_type_datetime(field, line)
+        if field[:format]
+          DateTime.strptime(line.strip, field[:format])
+        else
+          DateTime.parse(line)
+        end
+      end
+
+      def parse_field_type_float(_, line)
+        line.to_f
+      end
+
+      def parse_field_type_decimal(_, line)
+        BigDecimal(line)
+      end
+
+      def parse_field_type_string(_, line)
+        line.strip
+      end
+
+      def parse_field_type_boolean(_, line)
+        line.strip == 'true'
+      end
+
+      # @abstract
+      class Config
+        def initialize(&block)
+          instance_eval(&block)
+        end
+
+        def url(url)
+          @url = url
+        end
+
+        def primary_key(key)
+          @primary_key = key
+        end
+
+        def to_h
+          {
+            url: @url,
+            primary_key: @primary_key
+          }
+        end
+      end
+    end
+  end
+end

data/lib/any_query/adapters/csv.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module AnyQuery
+  module Adapters
+    class Csv < Base
+      class Config < Base::Config
+        def to_h
+          {
+            url: @url,
+            primary_key: @primary_key,
+            table: @table
+          }
+        end
+      end
+
+      def parse_fields(model)
+        CSV.foreach(url, headers: true).map do |line|
+          result = {}
+          model.fields.each do |name, field|
+            result[name] = parse_field(field, line[field[:source] || name.to_s])
+          end
+          result
+        end
+      end
+
+      def url
+        @config[:url]
+      end
+
+      def load(model, select:, joins:, where:, limit:)
+        chain = parse_fields(model)
+        chain = fallback_where(chain, where) if where.present?
+        chain = chain.first(limit) if limit.present?
+        chain = resolve_joins(chain, joins) if joins.present?
+
+        chain.map! { |row| instantiate_model(model, row) }
+        chain = resolve_select(chain, select) if select.present?
+
+        chain
+      end
+
+      def resolve_joins(data, joins)
+        joins.map do |join|
+          resolve_join(data, join)
+        end
+        data
+      end
+
+      def parse_field(field, line)
+        result = parse_field_type(field, line)
+        if field[:transform]
+          field[:transform].call(result)
+        else
+          result
+        end
+      end
+    end
+  end
+end

data/lib/any_query/adapters/fixed_length.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  module Adapters
+    # @api private
+    class FixedLength < Base
+      # @api private
+      class Config < Base::Config
+        def to_h
+          {
+            url: @url,
+            primary_key: @primary_key,
+            table: @table
+          }
+        end
+      end
+
+      def initialize(config)
+        super(config)
+        @file = File.open(url)
+      end
+
+      def parse_fields(model)
+        @file.each_line.map do |line|
+          result = {}
+          last_index = 0
+          model.fields.each do |name, field|
+            raw_value = line[last_index...(last_index + field[:length])]
+            result[name] = parse_field(field, raw_value)
+            last_index += field[:length]
+          end
+          result
+        end
+      end
+
+      def url
+        @config[:url]
+      end
+
+      def load(model, select:, joins:, where:, limit:)
+        @file.rewind
+
+        chain = parse_fields(model)
+        chain = fallback_where(chain, where) if where.present?
+        chain = chain.first(limit) if limit.present?
+        chain = resolve_joins(chain, joins) if joins.present?
+
+        chain.map! { |row| instantiate_model(model, row) }
+        chain = resolve_select(chain, select) if select.present?
+
+        chain
+      end
+
+      def resolve_joins(data, joins)
+        joins.map do |join|
+          resolve_join(data, join)
+        end
+        data
+      end
+
+      def parse_field(field, line)
+        result = parse_field_type(field, line)
+        if field[:transform]
+          field[:transform].call(result)
+        else
+          result
+        end
+      end
+    end
+  end
+end

data/lib/any_query/adapters/http.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  module Adapters
+    # @api private
+    class Http < Base
+      MAX_ITERATIONS = 1000
+      # @api private
+      class Config < Base::Config
+        def endpoint(name, method, path, options = {})
+          @endpoints ||= {}
+          @endpoints[name] = { method:, path:, options: }
+        end
+
+        def to_h
+          {
+            url: @url,
+            primary_key: @primary_key,
+            wrapper: @wrapper,
+            endpoints: @endpoints
+          }
+        end
+      end
+
+      def load(model, select:, joins:, where:, limit:)
+        data = run_http_list_query(where)
+
+        data = resolve_joins(data, joins) if joins.present?
+        data = data.first(limit) if limit.present?
+
+        parse_response(model, select, data)
+      end
+
+      def load_single(model, id, joins)
+        data = run_http_single_query(id, {})
+
+        data = resolve_joins(data, joins) if joins.present?
+
+        instantiate_model(model, data)
+      end
+
+      def parse_response(model, select, data)
+        data = data.map do |record|
+          instantiate_model(model, record)
+        end
+        data = resolve_select(data, select) if select.present?
+
+        data
+      end
+
+      # FIXME: Use common method
+      def load_single_from_list(data)
+        data.each_slice(50).flat_map do |slice|
+          slice
+            .map { |data| Thread.new { run_http_single_query(data[:id], {}) } }
+            .each(&:join)
+            .map(&:value)
+        end
+      end
+
+      def resolve_joins(data, joins)
+        data = load_single_from_list(data) if joins.any? { |j| j[:model] == :show }
+
+        joins.each do |join|
+          next if join[:model] == :show
+
+          resolve_join(data, join)
+        end
+        data
+      end
+
+      def build_filters(where)
+        {
+          query: (where || {}).inject({}) do |memo, object|
+                   memo.merge(object)
+                 end
+        }
+      end
+
+      def run_http_single_query(id, params)
+        endpoint = @config[:endpoints][:show]
+        url = build_url(endpoint, params, id:)
+        params = (endpoint[:options][:default_params] || {}).merge(params)
+        AnyQuery::Config.logger.debug "Starting request to #{url} with params #{params.inspect}"
+
+        data = run_http_request(endpoint, url, params)
+        data = data.dig(*endpoint[:options][:wrapper]) if endpoint[:options][:wrapper]
+        AnyQuery::Config.logger.debug 'Responded with single record.'
+        data
+      end
+
+      def run_http_list_query(raw_params)
+        endpoint = @config[:endpoints][:list]
+        url = build_url(endpoint, raw_params)
+        params = build_filters(raw_params)
+        results = Set.new
+        previous_response = nil
+        MAX_ITERATIONS.times do |i|
+          params = merge_params(endpoint, params, i, previous_response)
+
+          AnyQuery::Config.logger.debug "Starting request to #{url} with params #{params.inspect}"
+
+          data = run_http_request(endpoint, url, params)
+          break if previous_response == data
+
+          previous_response = data
+
+          data = unwrap(endpoint, data)
+
+          AnyQuery::Config.logger.debug "Responded with #{data&.size || 0} records"
+          break if !data || data.empty?
+
+          previous_count = results.size
+          results += data
+          break if results.size == previous_count
+
+          break if endpoint.dig(:options, :pagination, :type) == :none
+        end
+        results.to_a
+      end
+
+      def merge_params(endpoint, params, iteration, previous_response)
+        (endpoint[:options][:default_params] || {})
+          .deep_merge(params)
+          .deep_merge(handle_pagination(endpoint, iteration, previous_response))
+      end
+
+      def build_url(endpoint, params, id: nil)
+        output = (@config[:url] + endpoint[:path])
+
+        output.gsub!('{id}', id.to_s) if id
+
+        if output.include?('{')
+          output.gsub!(/\{([^}]+)\}/) do |match|
+            key = Regexp.last_match(1).to_sym
+            hash = params.find { |h| h[Regexp.last_match(1).to_sym] }
+            hash&.delete(key) || match
+          end
+        end
+
+        output
+      end
+
+      def unwrap(endpoint, data)
+        data = unwrap_list(endpoint, data)
+        unwrap_single(endpoint, data)
+      end
+
+      def unwrap_list(endpoint, data)
+        wrapper = endpoint[:options][:wrapper]
+        return data unless wrapper
+
+        if wrapper.is_a?(Proc)
+          wrapper.call(data)
+        else
+          data.dig(*wrapper)
+        end
+      end
+
+      def unwrap_single(endpoint, data)
+        return data unless endpoint[:options][:single_wrapper]
+
+        data.map! do |row|
+          row.dig(*endpoint[:options][:single_wrapper])
+        end
+      end
+
+      def run_http_request(endpoint, url, params)
+        response = HTTParty.public_send(endpoint[:method], url, params)
+
+        raise response.inspect unless response.success?
+
+        if response.parsed_response.is_a?(Array)
+          response.parsed_response.map(&:deep_symbolize_keys)
+        else
+          response.parsed_response&.deep_symbolize_keys
+        end
+      end
+
+      def handle_pagination(endpoint, index, previous_response = nil)
+        pagination = endpoint.dig(:options, :pagination) || {}
+        method_name = "handle_pagination_#{pagination[:type]}"
+        if respond_to?(method_name, true)
+          send(method_name, pagination, index, previous_response)
+        else
+          AnyQuery::Config.logger.warn "Unknown pagination type #{pagination[:type]}"
+          { query: { page: index } }
+        end
+      end
+
+      def handle_pagination_page(pagination, index, _previous_response)
+        starts_from = pagination[:starts_from] || 0
+        { query: { (pagination.dig(:params, :number) || :page) => starts_from + index } }
+      end
+
+      def handle_pagination_skip(_pagination, _index, _previous_response)
+        raise 'TODO: Implement skip pagination'
+      end
+
+      def handle_pagination_cursor(pagination, _index, previous_response)
+        return {} unless previous_response
+
+        cursor_parameter = pagination.dig(:params, :cursor) || :cursor
+        cursor = previous_response[cursor_parameter]
+        { query: { (pagination.dig(:params, :cursor) || :cursor) => cursor } }
+      end
+
+      def handle_pagination_none(_pagination, _index, _previous_response)
+        {}
+      end
+    end
+  end
+end

data/lib/any_query/adapters/sql.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  module Adapters
+    class Sql < Base
+      class Config < Base::Config
+        def table(name)
+          @table = name
+        end
+
+        def to_h
+          {
+            url: @url,
+            primary_key: @primary_key,
+            table: @table
+          }
+        end
+      end
+
+      def initialize(config)
+        super(config)
+        ActiveRecord::Base.establish_connection(@config[:url])
+        table_name = @config[:table]
+
+        @rails_model = declare_model!
+        @rails_model.table_name = table_name
+        @rails_model.inheritance_column = :_sti_disabled
+        Object.const_set("AnyQuery#{table_name.classify}", @rails_model)
+      end
+
+      def declare_model!
+        Class.new(ActiveRecord::Base) do
+          def dig(key, *other)
+            data = public_send(key)
+            return data if other.empty?
+
+            return unless data.respond_to?(:dig)
+
+            data.dig(*other)
+          end
+        end
+      end
+
+      attr_reader :rails_model
+
+      def url
+        @config[:url]
+      end
+
+      def load(_model, select:, joins:, where:, limit:)
+        declare_required_associations!(joins)
+        chain = @rails_model.all
+        chain = chain.where(*where) if where.present?
+        chain = chain.limit(limit) if limit.present?
+        chain = resolve_joins(chain, joins) if joins.present?
+
+        chain = resolve_select(chain, select) if select.present?
+
+        chain
+      end
+
+      def declare_required_associations!(joins)
+        joins&.each do |join|
+          next if join[:model]._adapter.url != @config[:url]
+
+          relation = join_relation_name(join)
+
+          if join[:as] == :list
+            @rails_model.has_many(relation, class_name: join[:model]._adapter.rails_model.to_s,
+                                            foreign_key: join[:foreign_key], primary_key: join[:primary_key])
+          else
+            @rails_model.belongs_to(relation, class_name: join[:model]._adapter.rails_model.to_s,
+                                              foreign_key: join[:foreign_key], primary_key: join[:primary_key])
+          end
+        end
+      end
+
+      def resolve_joins(data, joins)
+        joins.map do |join|
+          if join[:model]._adapter.url == @config[:url]
+            relation = join_relation_name(join)
+
+            data = data.eager_load(relation)
+          else
+            resolve_join(data, join)
+          end
+        end
+        data
+      end
+
+      def join_relation_name(join)
+        if join[:as] == :list
+          join[:model].table_name.pluralize.to_sym
+        else
+          join[:model].table_name.singularize.to_sym
+        end
+      end
+    end
+  end
+end

data/lib/any_query/adapters.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  # @api private
+  module Adapters
+    extend ActiveSupport::Autoload
+    autoload :Base
+    autoload :Http
+    autoload :Sql
+    autoload :FixedLength
+    autoload :Csv
+  end
+end

data/lib/any_query/config.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module AnyQuery
+  # A class to handle configuration for AnyQuery
+  class Config
+    attr_writer :logger
+
+    def self.logger
+      @logger ||= Logger.new($stdout)
+    end
+  end
+end

data/lib/any_query/field.rb

@@ -0,0 +1,7 @@
+module AnyQuery
+  class Field
+    attr_reader :name, :type, :options
+
+    def initialize(name, type, options = {}); end
+  end
+end