card-mod-lookup 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b4215382b1478803a91069e8f71e7a980a64d632c2d5b62e1bd69cf5fb0c7cbe
+  data.tar.gz: 746772c19ae1fa95a3d685a8f8befd8d2d2b516033908d99ae82a7ce0175eeed
+SHA512:
+  metadata.gz: 03eef8d51212c24dffb854ea9d1727c8accd0cb1371a4d46f0b3c05ae478b209c3bd3cd86946ee9a4637e18513126fc342b26d9a7395f5656baaa45972df8c07
+  data.tar.gz: 3eab763cc4378322b543659203326ab436a4990b6ec8be550e5b2b97bd2ed4714c74e54db61c310b9eae44cb6e8573d426f4ee45baa0a370c53ebcd7f686ca3f

data/README.md

@@ -0,0 +1,118 @@
+<!--
+# @title README - mod: lookup
+-->
+
+# Lookup mod
+
+This mod manages lookup tables that help decko sites scale as their decks
+grow increasingly large and their queries increasingly complex.
+
+## Why
+
+CQL is a powerful language for navigating card relationships. It is easy, for
+example, to find all the cards of a given type that have a field card of a given
+name that link to a third card with given content and on and on and on.
+
+But to work, these CQL queries must be translated into SQL and executed, 
+and when a database grows large, it can become quite expensive to execute all
+the implied self joins.
+
+For example, consider the use case that first inspired this code: _answers_ on
+WikiRate.org. WikiRate is a site that collects answers to questions about 
+companies. A given answer is a response to a given metric for a given year
+for a given company. When you consider all the kinds of companies and metrics
+and metric designers that WikiRate supports, you can imagine the queries 
+becoming quite complex if we have to re-join the cards table to itself every
+time we want to consider a different variable.
+
+The solution was to make a "lookup" table for answers that employs much more
+conventional relational database design. 
+
+_An increasingly common Decko pattern is to design structures organically and
+fluidly with cards and then to optimize those structures with lookup tables once
+the structures have matured. This pattern can be a surprisingly pleasant change
+to those accustomed to having to try to perfect their data structure before
+they've collected any data._
+
+## How
+
+### LookupTable
+
+To create a lookup table, you will need to create a database table (most
+often by using Rails migrations) and a ruby class like the following `Country`
+lookup:
+
+in lib/country.rb:
+```
+# class for lookup table named "countries"
+class Country < Cardio::Record
+
+  # country_id in countries table corresponds to id of company card
+  @card_column = :country_id
+  
+  # query of all cards in lookup
+  @card_query = { type_id: :company, trash: false }
+
+  include LookupTable
+  
+  # The following three are equivalent.
+  # Each would populate a `continent_id` column based on a value returned 
+  # by the continent_id method on the country card.
+  
+  # explicit fetch method
+  def fetch_continent_id
+    card.continent_d
+  end
+  
+  # fetcher with hash argument. hash value becomes method
+  fetcher continent_id: :continent_id
+  
+  # fetcher with symbol arguments. column name and method must be the same
+  fetcher :continent_id
+  
+```
+
+### Abstract::Lookup
+
+in set/type/country.rb:
+```
+# methods for accessing lookup entries
+include_set Abstract::Lookup
+
+# events for maintaining lookup sync
+include_set Abstract::LookupEvents
+
+# record class from above
+def lookup_class
+  ::Country 
+end
+
+# called by lookup instance; uses cards
+def continent_id
+  fetch(:continent)&.id
+end
+
+# uses lookup table
+def continent_id_from_lookup
+  lookup.continent_id
+end
+
+```
+
+### Abstract::LookupField
+
+in set/right/continent.rb
+```
+# methods for maintaing companies table when continent changes
+# (admittedly an infrequent occurence)
+include_set Abstract::LookupField
+```
+
+
+### Abstract::LookupSearch
+
+Include this set to gain a helpful framework for developing a filter interface
+for lookup tables.
+
+
+

data/lib/card/lookup_filter_query/active_record_extension.rb

@@ -0,0 +1,43 @@
+class Card
+  class LookupFilterQuery
+    # support methods for sort and page
+    module ActiveRecordExtension
+      # @params hash [Hash] key1: dir1, key2: dir2
+      def sort hash
+        hash.present? ? sort_by_hash(hash) : self
+      end
+
+      def paging args
+        return self unless valid_page_args? args
+        limit(args[:limit]).offset(args[:offset])
+      end
+
+      private
+
+      def valid_page_args? args
+        args.present? && args[:limit].to_i.positive?
+      end
+
+      def sort_by_hash hash
+        rel = self
+        hash.each do |fld, dir|
+          rel, fld = interpret_sort_field rel, fld
+          rel = rel.order Arel.sql("#{fld} #{dir}")
+        end
+        rel
+      end
+
+      def interpret_sort_field rel, fld
+        if (match = fld.match(/^(\w+)_bookmarkers$/))
+          sort_by_bookmarkers match[1], rel
+        else
+          [rel, fld]
+        end
+      end
+
+      def sort_by_bookmarkers type, rel
+        [Card::Bookmark.add_sort_join(rel, "#{table.name}.#{type}_id"), "cts.value"]
+      end
+    end
+  end
+end

data/lib/card/lookup_filter_query/filtering.rb

@@ -0,0 +1,98 @@
+class Card
+  class LookupFilterQuery
+    # shared filtering methods for FilterQuery classes built on lookup tables
+    module Filtering
+      def process_filters
+        normalize_filter_args
+        return if @empty_result
+        @filter_args.each { |k, v| process_filter_option k, v if v.present? }
+        @restrict_to_ids.each { |k, v| filter k, v }
+      end
+
+      def normalize_filter_args
+        # override
+      end
+
+      def process_filter_option key, value
+        if (method = filter_method key)
+          send method, key, value
+        else
+          try "#{key}_query", value
+        end
+      end
+
+      def filter_method key
+        case key
+        when *simple_filters
+          :filter_exact_match
+        when *card_id_filters
+          :filter_card_id
+        end
+      end
+
+      def filter_exact_match key, value
+        filter key, value if value.present?
+      end
+
+      def filter_card_id key, value
+        return unless (card_id = to_card_id value)
+
+        filter card_id_map[key], card_id
+      end
+
+      def not_ids_query value
+        add_condition "#{lookup_class.card_column} not in (?)", value.split(",")
+      end
+
+      def to_card_id value
+        if value.is_a?(Array)
+          value.map { |v| Card.fetch_id(v) }
+        else
+          Card.fetch_id(value)
+        end
+      end
+
+      def restrict_to_ids col, ids
+        ids = Array(ids)
+        @empty_result ||= ids.empty?
+        restrict_lookup_ids col, ids
+      end
+
+      def restrict_lookup_ids col, ids
+        existing = @restrict_to_ids[col]
+        @restrict_to_ids[col] = existing ? (existing & ids) : ids
+      end
+
+      def restrict_by_cql col, cql
+        cql.reverse_merge! return: :id, limit: 0
+        restrict_to_ids col, Card.search(cql)
+      end
+
+      def filter field, value, operator=nil
+        condition = "#{filter_table field}.#{field} #{op_and_val operator, value}"
+        add_condition condition, value
+      end
+
+      def filter_table _field
+        lookup_table
+      end
+
+      def op_and_val op, val
+        "#{db_operator op, val} #{db_value val}"
+      end
+
+      def add_condition condition, value
+        @conditions << condition
+        @values << value
+      end
+
+      def db_operator operator, value
+        operator || (value.is_a?(Array) ? "IN" : "=")
+      end
+
+      def db_value value
+        value.is_a?(Array) ? "(?)" : "?"
+      end
+    end
+  end
+end

data/lib/card/lookup_filter_query.rb

@@ -0,0 +1,115 @@
+class Card
+  # base class for FilterQuery classes built on lookup tables
+  class LookupFilterQuery
+    include Filtering
+
+    attr_accessor :filter_args, :sort_args, :paging_args
+    class_attribute :card_id_map, :card_id_filters, :simple_filters
+
+    def initialize filter, sorting={}, paging={}
+      @filter_args = filter
+      @sort_args = sorting
+      @paging_args = paging
+
+      @conditions = []
+      @joins = []
+      @values = []
+      @restrict_to_ids = {}
+
+      process_sort
+      process_filters
+    end
+
+    def lookup_query
+      q = lookup_class.where lookup_conditions
+      q = q.joins(@joins.uniq) if @joins.present?
+      q
+    end
+
+    def lookup_table
+      @lookup_table ||= lookup_class.arel_table.name
+    end
+
+    def condition_sql conditions
+      lookup_class.sanitize_sql_for_conditions conditions
+    end
+
+    def lookup_relation
+      sort_and_page { lookup_query }
+    end
+
+    # @return args for AR's where method
+    def lookup_conditions
+      condition_sql([@conditions.join(" AND ")] + @values)
+    end
+
+    # TODO: support optionally returning lookup objects
+
+    # @return array of metric answer card objects
+    #   if filtered by missing values then the card objects
+    #   are newly instantiated and not in the database
+    def run
+      @empty_result ? [] : main_results
+    end
+
+    # @return [Array]
+    def count
+      @empty_result ? 0 : main_query.count
+    end
+
+    def limit
+      @paging_args[:limit]
+    end
+
+    def main_query
+      @main_query ||= lookup_query
+    end
+
+    def main_results
+      # puts "SQL: #{lookup_relation.to_sql}"
+      lookup_relation.map(&:card)
+    end
+
+    private
+
+    def sort_and_page
+      relation = yield
+      @sort_joins.uniq.each { |j| relation = relation.joins(j) }
+
+      relation.sort(@sort_hash).paging(@paging_args)
+    end
+
+    def process_sort
+      @sort_joins = []
+      @sort_hash = @sort_args.each_with_object({}) do |(by, dir), h|
+        h[sort_by(by)] = sort_dir(dir)
+      end
+    end
+
+    def sort_by sort_by
+      if (id_field = sort_by_cardname[sort_by])
+        sort_by_join sort_by, lookup_table, id_field
+      else
+        simple_sort_by sort_by
+      end
+    end
+
+    def sort_by_cardname
+      {}
+    end
+
+    def sort_dir dir
+      dir
+    end
+
+    def simple_sort_by sort_by
+      sort_by
+    end
+
+    def sort_by_join sort_by, from_table, from_id_field
+      @sort_joins <<
+        "JOIN cards as #{sort_by} ON #{sort_by}.id = #{from_table}.#{from_id_field}"
+      "#{sort_by}.key"
+    end
+  end
+end

data/lib/lookup_table/class_methods.rb

@@ -0,0 +1,104 @@
+module LookupTable
+  # shared class methods for lookup tables.
+  module ClassMethods
+    attr_reader :card_column,
+                :card_query # cql that finds all items in the cards table
+
+    def for_card cardish
+      card_id = Card.id cardish
+      card_id ? where(card_column => card_id).take : nil
+    end
+    alias_method :find_for_card, :for_card
+
+    def new_for_card cardish
+      new.tap do |lookup|
+        lookup.card_id = Card.id cardish
+      end
+    end
+
+    # TODO: change to create_for_card for consistency
+    def create cardish
+      new_for_card(cardish).refresh
+    end
+
+    def create! cardish
+      lookup = new_for_card cardish
+      raise ActiveRecord::RecordInvalid, lookup if lookup.invalid?
+      lookup.refresh
+    end
+
+    def create_or_update cardish, *fields
+      lookup = for_card(cardish) || new_for_card(cardish)
+      fields = nil if lookup.new_record? # update all fields if record is new
+      lookup.refresh(*fields)
+    end
+
+    # @param ids [Array<Integer>] ids of answers in the answer table (NOT card ids)
+    def update_by_ids ids, *fields
+      Array(ids).each do |id|
+        next unless (entry = find_by_id(id))
+        entry.refresh(*fields)
+      end
+    end
+
+    def delete_for_card cardish
+      for_card(cardish)&.destroy
+    end
+
+    # @param ids [Integer, Array<Integer>] card ids of metric answer cards
+    def refresh ids=nil, *fields
+      Array(ids).compact.each do |ma_id|
+        refresh_entry fields, ma_id
+      end
+    end
+
+    def refresh_entry fields, card_id
+      if Card.exists? card_id
+        create_or_update card_id, *fields
+      else
+        delete_for_card card_id
+      end
+    rescue StandardError => e
+      raise e, "failed to refresh #{name} lookup table " \
+               "for card id #{card_id}: #{e.message}"
+    end
+
+    def refresh_all fields=nil
+      count = 0
+      Card.where(card_query).pluck_in_batches(:id) do |batch|
+        count += batch.size
+        puts "#{batch.first} - #{count}"
+        refresh(batch, *fields)
+      end
+    end
+
+    # define standard lookup fetch methods.
+    #
+    # Eg. fetcher(:company_id) defines #fetch_company_id to fetch from card.company_id
+    #
+    # And fetcher(foo: :bar) defines #fetch_foo to fetch from card.bar
+    def fetcher *args
+      fetcher_hash(*args).each { |col, method| define_fetch_method col, method }
+    end
+
+    def define_main_fetcher
+      define_fetch_method @card_column, :id
+    end
+
+    private
+
+    def define_fetch_method column, card_method
+      define_method "fetch_#{column}" do
+        card.send card_method
+      end
+    end
+
+    def fetcher_hash *args
+      if args.first.is_a?(Hash)
+        args.first
+      else
+        args.each_with_object({}) { |item, h| h[item] = item }
+      end
+    end
+  end
+end

data/lib/lookup_table.rb

@@ -0,0 +1,64 @@
+# lookup table to optimize complex card systems
+#
+# TODO: make this a class and have lookup classes inherit from it
+module LookupTable
+  def self.included host_class
+    host_class.extend LookupTable::ClassMethods
+    host_class.define_main_fetcher
+  end
+
+  def card_column
+    self.class.card_column
+  end
+
+  def card
+    @card ||= Card.fetch send(card_column), look_in_trash: true
+  end
+
+  def card_id
+    send card_column
+  end
+
+  def card_id= id
+    send "#{card_column}=", id
+  end
+
+  def delete_on_refresh?
+    !card || card.trash
+  end
+
+  def refresh *fields
+    return delete if delete_on_refresh?
+
+    refresh_fields fields
+    raise Card::Error, "invalid #{self.class} lookup" if invalid?
+
+    save!
+  end
+
+  def refresh_fields fields=nil
+    keys = fields.present? ? fields : attribute_names
+    keys.delete("id")
+    keys.each { |method_name| refresh_value method_name }
+  end
+
+  def refresh_value method_name
+    send "#{method_name}=", send("fetch_#{method_name}")
+  end
+
+  def method_missing method_name, *args, &block
+    if card.respond_to? method_name
+      card.send method_name, *args, &block
+    else
+      super
+    end
+  end
+
+  def respond_to_missing? *args
+    card.respond_to?(*args) || super
+  end
+
+  def is_a? klass
+    klass == Card || super
+  end
+end

data/set/abstract/lookup.rb

@@ -0,0 +1,7 @@
+def lookup_class
+  raise StandardError, "must define #lookup_class"
+end
+
+def lookup
+  lookup_class.for_card id
+end

data/set/abstract/lookup_events.rb

@@ -0,0 +1,12 @@
+event :create_lookup, :finalize, on: :create do
+  lookup_class.create self
+end
+
+# lookup fields are often based on cards' compound names
+event :refresh_lookup, :integrate, changed: :name, on: :update do
+  lookup.refresh
+end
+
+event :delete_lookup, :finalize, on: :delete do
+  lookup_class.delete_for_card id
+end

data/set/abstract/lookup_field.rb

@@ -0,0 +1,23 @@
+def lookup_card
+  left
+end
+
+def lookup
+  lookup_card.lookup
+end
+
+def lookup_columns
+  Codename[right_id]
+end
+
+event :update_lookup_field, :finalize, changed: :content do
+  lookup_field_update do
+    lookup.refresh(*Array.wrap(lookup_columns))
+  end
+end
+
+private
+
+def lookup_field_update
+  yield unless lookup_card.action.in? %i[create delete]
+end

data/set/abstract/lookup_search.rb

@@ -0,0 +1,94 @@
+def search args={}
+  return_type = args.delete :return
+  query = args.delete(:query) || query(args)
+  run_query_returning query, return_type
+end
+
+def run_query_returning query, return_type
+  case return_type
+  when :name  then query.run.map(&:name)
+  when :count then query.count
+  else             query.run
+  end
+end
+
+def filter_class
+  raise Error::ServerError, "filter_class required!"
+end
+
+def query paging={}
+  filter_class.new query_hash, {}, paging
+end
+
+def query_hash
+  {}
+end
+
+def count
+  query.count
+end
+
+format do
+  delegate :filter_class, to: :card
+
+  def search_with_params
+    @search_with_params ||= card.search query: query
+  end
+
+  def count_with_params
+    @count_with_params ||= card.search query: count_query, return: :count
+  end
+
+  def query
+    filter_class.new query_hash, sort_hash, paging_params
+  end
+
+  def count_query
+    filter_class.new query_hash
+  end
+
+  def query_hash
+    (filter_hash || {}).merge card.query_hash
+  end
+
+  def default_filter_hash
+    card.query_hash
+  end
+
+  def sort_hash
+    { sort_by.to_sym => sort_dir }
+  end
+
+  def sort_dir
+    return unless sort_by
+    @sort_dir ||= safe_sql_param("sort_dir") || default_sort_dir(sort_by)
+  end
+
+  def default_sort_dir sort_by
+    if default_desc_sort_dir.include? sort_by.to_sym
+      :desc
+    else
+      :asc
+    end
+  end
+
+  def default_desc_sort_dir
+    []
+  end
+
+  def sort_by
+    @sort_by ||= sort_by_from_param || default_sort_option
+  end
+
+  def default_sort_option
+    # override
+  end
+
+  def sort_by_from_param
+    safe_sql_param(:sort_by)&.to_sym
+  end
+
+  def default_limit
+    Auth.signed_in? ? 5000 : 500
+  end
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: card-mod-lookup
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Philipp Kühl
+- Ethan McCutchen
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-10-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: card
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- info@decko.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/card/lookup_filter_query.rb
+- lib/card/lookup_filter_query/active_record_extension.rb
+- lib/card/lookup_filter_query/filtering.rb
+- lib/lookup_table.rb
+- lib/lookup_table/class_methods.rb
+- set/abstract/lookup.rb
+- set/abstract/lookup_events.rb
+- set/abstract/lookup_field.rb
+- set/abstract/lookup_search.rb
+homepage: http://decko.org
+licenses:
+- GPL-3.0
+metadata:
+  card-mod: lookup
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.28
+signing_key:
+specification_version: 4
+summary: lookup
+test_files: []