keinaufwand-sync 0.1.0

data/lib/keinaufwand/sync_core/record.rb

@@ -0,0 +1,410 @@
+module Keinaufwand
+  module Sync
+    class Record < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name = "keinaufwand_records"
+      self.primary_key = [:id, :type]
+
+      class_attribute :keinaufwand_schema, instance_writer: false
+
+      def self.configure_keinaufwand_schema!(schema)
+        self.keinaufwand_schema = schema
+        descendants.each(&:boot_keinaufwand_dynamics!) if schema && respond_to?(:descendants)
+      end
+
+      def self.where_present(*attributes)
+        attributes.reduce(all) do |scope, attribute|
+          expression = json_value(attribute)
+          scope.where("#{expression} IS NOT NULL").where("#{expression} != ?", "")
+        end
+      end
+
+      def self.find(*ids)
+        return super if ids.empty?
+
+        ids = ids.flatten
+        return where(id: ids).index_by(&:id).values_at(*ids.map(&:to_i)) if ids.many?
+
+        find_by!(id: ids.first)
+      end
+
+      def self.exists?(conditions = :none)
+        if composite_primary_key? && conditions != :none && !conditions.is_a?(Hash) && !conditions.is_a?(Array)
+          return super(id: conditions)
+        end
+
+        super
+      end
+
+      def self.json_value(attribute)
+        table = connection.quote_table_name(table_name)
+        column = connection.quote_column_name("data")
+        json_value_expression("#{table}.#{column}", attribute)
+      end
+
+      def self.json_value_expression(column, attribute)
+        if connection.adapter_name.match?(/postgres/i)
+          expression = "#{column} #>> #{connection.quote("{#{attribute.to_s.split(".").join(",")}}")}"
+          postgres_cast(expression, attribute)
+        else
+          expression = "json_extract(#{column}, #{connection.quote("$.#{attribute}")})"
+          sqlite_cast(expression, attribute)
+        end
+      end
+
+      def self.json_attribute?(attribute)
+        !column_names.include?(attribute.to_s) && schema_attribute?(attribute)
+      end
+
+      def self.schema
+        keinaufwand_schema || load_keinaufwand_schema
+      end
+
+      def self.load_keinaufwand_schema
+        version = Keinaufwand::Api.configuration.version || "2026-07-02"
+        self.keinaufwand_schema = Keinaufwand::Api::Schema.load(Keinaufwand::Api.configuration.schema_path(version))
+      end
+
+      def self.model_schema
+        schema&.for(name.demodulize)
+      end
+
+      def self.schema_attributes
+        model_schema&.fetch("attributes", {}) || {}
+      end
+
+      def self.schema_associations
+        model_schema&.fetch("associations", {}) || {}
+      end
+
+      def self.schema_attribute?(attribute)
+        attribute = attribute.to_s
+        schema_attributes.key?(attribute) || schema_attributes.key?(attribute.split(".").first)
+      end
+
+      def self.schema_attribute_type(attribute)
+        attribute = attribute.to_s
+        (schema_attributes[attribute].presence || schema_attributes[attribute.split(".").first]).to_s
+      end
+
+      def self.inherited(subclass)
+        super
+        subclass.boot_keinaufwand_dynamics!
+      end
+
+      def self.boot_keinaufwand_dynamics!
+        return if abstract_class? || name.blank? || name == "Keinaufwand::Record"
+
+        schema_attributes.each_key do |attribute|
+          define_dynamic_attribute(attribute)
+        end
+
+        schema_associations.each do |association, config|
+          define_dynamic_association(association, config)
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def self.define_dynamic_attribute(attribute)
+        attribute = attribute.to_s
+        return if attribute == "id"
+
+        define_method(attribute) do |variant = :large|
+          value = read_keinaufwand_value(attribute)
+          if asset_attribute?(attribute, value)
+            value = derivative_value(value, variant)
+            value = local_asset_url(value) if respond_to?(:local_asset_url)
+          end
+          value = localize_asset_urls(value) if attribute == "body" && respond_to?(:localize_asset_urls)
+          cast_keinaufwand_value(attribute, value)
+        end
+
+        define_method("#{attribute}?") do
+          value = public_send(attribute)
+          value == true || value.present?
+        end
+      end
+
+      def self.define_dynamic_association(association, config)
+        association = association.to_s
+        config = config.with_indifferent_access
+        type = config[:type].to_s
+        target_class_name = config[:class_name] || association.singularize.classify
+
+        define_dynamic_foreign_key(association, type) if %w[belongs_to has_one].include?(type)
+        define_dynamic_ids(association) if type == "has_and_belongs_to_many"
+
+        define_method(association) do
+          return instance_variable_get("@#{association}") if instance_variable_defined?("@#{association}")
+
+          target_class = target_class_name.safe_constantize
+          result = target_class ? load_keinaufwand_association(association, type, target_class, config) : nil
+          instance_variable_set("@#{association}", result)
+        end
+
+        define_method("#{association}=") do |value|
+          instance_variable_set("@#{association}", value)
+        end
+      end
+
+      def self.define_dynamic_foreign_key(association, type)
+        key = "#{association}_id"
+        return if schema_attribute?(key)
+
+        define_method(key) do
+          read_keinaufwand_value(key)
+        end
+      end
+
+      def self.define_dynamic_ids(association)
+        key = "#{association.singularize}_ids"
+        return if schema_attribute?(key)
+
+        define_method(key) do
+          Array(read_keinaufwand_value(key))
+        end
+      end
+
+      def self.postgres_cast(expression, attribute)
+        case schema_attribute_type(attribute)
+        when "integer" then "(#{expression})::integer"
+        when "float", "decimal" then "(#{expression})::numeric"
+        when "date" then "(#{expression})::date"
+        when "datetime", "time" then "(#{expression})::timestamp"
+        when "boolean" then "(#{expression})::boolean"
+        else expression
+        end
+      end
+
+      def self.sqlite_cast(expression, attribute)
+        case schema_attribute_type(attribute)
+        when "integer" then "CAST(#{expression} AS INTEGER)"
+        when "float", "decimal" then "CAST(#{expression} AS REAL)"
+        else expression
+        end
+      end
+
+      def self.json_hash_conditions?(conditions)
+        conditions.is_a?(Hash) && conditions.any? { |key, _| json_attribute?(key) }
+      end
+
+      def self.split_json_conditions(conditions)
+        conditions.partition { |key, _| json_attribute?(key) }.map(&:to_h)
+      end
+
+      def self.apply_json_conditions(relation, conditions)
+        conditions.reduce(relation) do |scope, (attribute, value)|
+          scope.where(json_condition_sql(attribute, value), *json_condition_binds(value))
+        end
+      end
+
+      def self.json_order_argument?(argument)
+        case argument
+        when Symbol
+          json_attribute?(argument)
+        when Hash
+          argument.keys.all? { |key| json_attribute?(key) }
+        else
+          false
+        end
+      end
+
+      def self.json_order_sql(argument)
+        case argument
+        when Symbol
+          json_value(argument)
+        when Hash
+          argument.map do |attribute, direction|
+            direction = direction.to_s.downcase == "desc" ? "DESC" : "ASC"
+            "#{json_value(attribute)} #{direction}"
+          end
+        end
+      end
+
+      def self.json_condition_sql(attribute, value)
+        expression = json_value(attribute)
+
+        case value
+        when Array
+          return "1=0" if value.empty?
+          return "(#{expression} IN (?) OR #{expression} IS NULL)" if value.include?(nil)
+
+          "#{expression} IN (?)"
+        when Range
+          return "#{expression} >= ?" if value.end.nil?
+          return "#{expression} #{value.exclude_end? ? '<' : '<='} ?" if value.begin.nil?
+          return "#{expression} >= ? AND #{expression} < ?" if value.exclude_end?
+
+          "#{expression} BETWEEN ? AND ?"
+        when nil
+          "#{expression} IS NULL"
+        else
+          "#{expression} = ?"
+        end
+      end
+
+      def self.json_condition_binds(value)
+        case value
+        when Array
+          return [] if value.empty?
+
+          [value.compact.map { |item| json_bind_value(item) }]
+        when Range
+          return [json_bind_value(value.begin)] if value.end.nil?
+          return [json_bind_value(value.end)] if value.begin.nil?
+
+          [json_bind_value(value.begin), json_bind_value(value.end)]
+        when nil
+          []
+        else
+          [json_bind_value(value)]
+        end
+      end
+
+      def self.json_bind_value(value)
+        case value
+        when Time, ActiveSupport::TimeWithZone
+          value.iso8601
+        when Date
+          value.iso8601
+        else
+          value
+        end
+      end
+
+      def id
+        read_attribute(:id)
+      end
+
+      def id=(value)
+        if value.is_a?(Array)
+          write_keinaufwand_column("id", value.first)
+          write_keinaufwand_column("type", value.second)
+        else
+          write_keinaufwand_column("id", value)
+          write_keinaufwand_column("type", self.class.name) if self[:type].blank? && self.class.name.present?
+        end
+      end
+
+      def primary_key_values_present?
+        Array(self.class.primary_key).all? { |column| @attributes.fetch_value(column).present? }
+      end
+
+      def write_keinaufwand_column(attribute, value)
+        @attributes.write_from_user(attribute, value)
+      end
+
+      def remember_transaction_record_state
+        super
+        return unless self.class.composite_primary_key? && @_start_transaction_state
+
+        @_start_transaction_state[:id] = self.class.primary_key.map { |column| @attributes.fetch_value(column) }
+      end
+
+      def read_keinaufwand_value(attribute)
+        data[attribute.to_s] if data.is_a?(Hash)
+      end
+
+      def load_keinaufwand_association(association, type, target_class, config)
+        inline_value = read_keinaufwand_value(association)
+
+        case type
+        when "belongs_to", "has_one"
+          target_id = read_keinaufwand_value("#{association}_id")
+          local = target_class.find_by(id: target_id) if target_id.present?
+          local || build_inline_keinaufwand_records(inline_value, target_class)
+        when "has_many"
+          foreign_key = config[:foreign_key] || "#{self.class.name.demodulize.underscore}_id"
+          local = target_class.where(foreign_key => id)
+          local.exists? ? local : build_inline_keinaufwand_records(inline_value, target_class)
+        when "has_and_belongs_to_many"
+          target_ids = Array(read_keinaufwand_value("#{association.singularize}_ids"))
+          target_class.where(id: target_ids)
+        else
+          build_inline_keinaufwand_records(inline_value, target_class)
+        end
+      end
+
+      def build_inline_keinaufwand_records(value, target_class)
+        if value.is_a?(Array)
+          value.map { |attributes| build_inline_keinaufwand_record(attributes, target_class) }.compact
+        elsif value.is_a?(Hash)
+          build_inline_keinaufwand_record(value, target_class)
+        end
+      end
+
+      def build_inline_keinaufwand_record(attributes, target_class)
+        return unless attributes.is_a?(Hash)
+
+        target_class.new(id: attributes["id"] || attributes[:id], type: target_class.name, data: attributes.except("id", :id))
+      end
+
+      def cast_keinaufwand_value(attribute, value)
+        case self.class.schema_attribute_type(attribute)
+        when "datetime", "time" then parse_keinaufwand_time(value) || value
+        when "date" then parse_keinaufwand_date(value) || value
+        when "integer" then value.to_i if value.present?
+        when "float" then value.to_f if value.present?
+        when "decimal" then value.to_d if value.present?
+        when "boolean" then value == true || value.to_s == "true"
+        else value
+        end
+      end
+
+      def parse_keinaufwand_time(value)
+        return value if value.is_a?(Time) || defined?(ActiveSupport::TimeWithZone) && value.is_a?(ActiveSupport::TimeWithZone)
+        return if value.blank?
+
+        Time.zone ? Time.zone.parse(value.to_s) : Time.parse(value.to_s)
+      rescue ArgumentError
+        nil
+      end
+
+      def parse_keinaufwand_date(value)
+        return value if value.is_a?(Date)
+        return if value.blank?
+
+        Date.parse(value.to_s)
+      rescue ArgumentError
+        nil
+      end
+
+      def asset_attribute?(attribute, value)
+        attribute.end_with?("_url") && value.is_a?(Hash)
+      end
+
+      def derivative_value(value, size)
+        return value unless value.is_a?(Hash)
+
+        value[size.to_s] || value[size.to_sym] || value.values.compact.first
+      end
+    end
+
+    module RelationMethods
+      def where(*args)
+        return super unless keinaufwand_record_relation? && args.first.is_a?(Hash) && klass.json_hash_conditions?(args.first)
+
+        json_conditions, column_conditions = klass.split_json_conditions(args.first)
+        relation = column_conditions.any? ? super(column_conditions, *args[1..]) : self
+        klass.apply_json_conditions(relation, json_conditions)
+      end
+
+      def order(*args)
+        return super unless keinaufwand_record_relation? && args.any? && args.all? { |arg| klass.json_order_argument?(arg) }
+
+        super(Arel.sql(args.flat_map { |arg| klass.json_order_sql(arg) }.join(", ")))
+      end
+
+      private
+
+      def keinaufwand_record_relation?
+        klass <= Keinaufwand::Sync::Record
+      rescue StandardError
+        false
+      end
+    end
+  end
+end
+
+ActiveRecord::Relation.prepend(Keinaufwand::Sync::RelationMethods)

data/lib/keinaufwand/sync_core/sync_service.rb

@@ -0,0 +1,86 @@
+module Keinaufwand
+  module Sync
+    class SyncService
+      DEFAULT_RESOURCES = %w[
+        Location Product Session EventDetail Event Post InstructorGroup
+      ].freeze
+
+      def initialize(client:, record_class:, sync_state_class:, resources: DEFAULT_RESOURCES, asset_class: nil, receipt_class: nil)
+        @client = client
+        @record_class = record_class
+        @sync_state_class = sync_state_class
+        @resources = resources
+        @asset_class = asset_class
+        @receipt_class = receipt_class
+      end
+
+      def perform
+        @resources.each { |resource| sync_resource(resource) }
+      end
+
+      private
+
+      def sync_resource(resource)
+        cursor = @sync_state_class.cursor_for(resource)
+        params = { limit: 100 }
+        params[:updated_since] = (cursor - 10.seconds).iso8601 if cursor
+
+        page = @client.public_send("retrieve_#{resource.pluralize.underscore}", params)
+        latest = nil
+
+        while page && !page.empty?
+          page.each do |record|
+            sync_assets(record.raw_attributes) if @asset_class && record.raw_attributes[:_deleted] != true
+          end
+
+          @record_class.transaction do
+            rows = page.filter_map do |record|
+              latest = latest_timestamp(latest, record.server_updated_at)
+              next if stale?(resource, record)
+
+              if record.raw_attributes[:_deleted] == true
+                delete_record(resource, record.id)
+                next
+              end
+
+              { id: record.id, type: resource, data: record.raw_attributes.except(:id), created_at: Time.current, updated_at: Time.current }
+            end
+
+            @record_class.upsert_all(rows, unique_by: %i[id type], update_only: %w[data updated_at]) if rows.any?
+          end
+          page = page.next
+        end
+
+        @sync_state_class.update_cursor!(resource, latest) if latest
+      end
+
+      def delete_record(resource, id)
+        @record_class.where(type: resource, id: id).delete_all
+      end
+
+      def stale?(resource, record)
+        return false unless record.server_updated_at
+
+        stored = @record_class.where(type: resource, id: record.id).pick(:data)
+        timestamp = stored&.fetch("updated_at", nil)
+        timestamps = [
+          (Time.iso8601(timestamp) if timestamp.present?),
+          @receipt_class&.latest_source_updated_at(resource, record.id)
+        ].compact
+        timestamps.any? { |value| value > record.server_updated_at }
+      end
+
+      def latest_timestamp(current, candidate)
+        candidate && (!current || candidate > current) ? candidate : current
+      end
+
+      def sync_assets(attributes)
+        @asset_class.urls_from(attributes).uniq.each do |url|
+          @asset_class.sync_url(url)
+        rescue StandardError => error
+          warn "Skipping Keinaufwand asset #{url}: #{error.message}"
+        end
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/sync_state.rb

@@ -0,0 +1,25 @@
+module Keinaufwand
+  module Sync
+    class SyncState < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name = "keinaufwand_sync_states"
+      self.primary_key = :resource_class
+
+      def self.inherited(subclass)
+        super
+        subclass.table_name = table_name
+        subclass.primary_key = primary_key
+      end
+
+      def self.cursor_for(resource_class)
+        find_by(resource_class: resource_class)&.last_synced_at
+      end
+
+      def self.update_cursor!(resource_class, timestamp)
+        state = find_or_initialize_by(resource_class: resource_class)
+        state.last_synced_at = timestamp
+        state.save!
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/tasks/keinaufwand_sync.rake

@@ -0,0 +1,19 @@
+namespace :keinaufwand do
+  desc "Sync Keinaufwand API records into local storage"
+  task sync: :environment do
+    Keinaufwand::Sync.sync!
+  end
+
+  desc "Clear local Keinaufwand records and run a fresh sync"
+  task resync: :environment do
+    Keinaufwand::Sync.resync!
+  end
+
+  namespace :indices do
+    desc "Generate a migration for Keinaufwand JSON indexes from config/keinaufwand.yml"
+    task generate: :environment do
+      require "rails/generators"
+      Rails::Generators.invoke("keinaufwand:sync:indexes", [], destination_root: Rails.root.to_s)
+    end
+  end
+end

data/lib/keinaufwand/sync_core/version.rb

@@ -0,0 +1,5 @@
+module Keinaufwand
+  module Sync
+    VERSION = "0.1.0"
+  end
+end

data/lib/keinaufwand/sync_core/webhook_processor.rb

@@ -0,0 +1,119 @@
+module Keinaufwand
+  module Sync
+    class WebhookProcessor
+      class InvalidPayload < StandardError; end
+
+      def initialize(payload:, record_class:, receipt_class:, resources:, asset_class: nil)
+        @payload = payload.with_indifferent_access
+        @record_class = record_class
+        @receipt_class = receipt_class
+        @resources = resources.map(&:to_s)
+        @asset_class = asset_class
+      end
+
+      def perform
+        validate_payload!
+        return :duplicate if @receipt_class.exists?(event_id: event_id)
+        return record_receipt!(:ignored) unless resource_type && @resources.include?(resource_type)
+        return record_receipt!(:stale) if stale?
+
+        result = @record_class.transaction do
+          deleted? ? delete_record : upsert_record
+          record_receipt!(deleted? ? :destroyed : :synced)
+        end
+        sync_assets unless deleted?
+        result
+      rescue ActiveRecord::RecordNotUnique
+        :duplicate
+      end
+
+      private
+
+      def validate_payload!
+        raise InvalidPayload, "event_id is required" if event_id.blank?
+        raise InvalidPayload, "event_type is required" if event_type.blank?
+        raise InvalidPayload, "api_version is required" if api_version.blank?
+        raise InvalidPayload, "data must be an object" unless data.is_a?(Hash)
+        raise InvalidPayload, "data.id is required" if resource_id.blank?
+        raise InvalidPayload, "unsupported API version #{api_version}" unless schema.version == api_version
+      end
+
+      def schema
+        @schema ||= Keinaufwand::Api::Schema.load(Keinaufwand::Api.configuration.schema_path(api_version))
+      end
+
+      def resource_type
+        @resource_type ||= schema.model_names.find do |model_name|
+          schema.for(model_name).fetch(:resource_name).singularize == event_type.split(".").first
+        end
+      end
+
+      def stale?
+        timestamps = [
+          @receipt_class.latest_source_updated_at(resource_type, resource_id),
+          existing_record_updated_at
+        ].compact
+        source_updated_at && timestamps.any? { |timestamp| timestamp > source_updated_at }
+      end
+
+      def existing_record_updated_at
+        value = local_record&.data&.fetch("updated_at", nil)
+        Time.iso8601(value) if value.present?
+      rescue ArgumentError
+        nil
+      end
+
+      def upsert_record
+        attributes = data.except(:id, :_deleted).to_h
+        record = local_record || @record_class.new(id: resource_id, type: resource_type)
+        record.data = attributes
+        record.save!
+      end
+
+      def delete_record
+        local_record&.delete
+      end
+
+      def record_receipt!(result)
+        @receipt_class.create!(
+          event_id: event_id,
+          event_type: event_type,
+          api_version: api_version,
+          resource_type: resource_type || event_type.split(".").first.classify,
+          resource_id: resource_id,
+          source_updated_at: source_updated_at,
+          result: result.to_s,
+          processed_at: Time.current
+        )
+        result
+      end
+
+      def sync_assets
+        return unless @asset_class
+
+        @asset_class.urls_from(data).uniq.each do |url|
+          @asset_class.sync_url(url)
+        rescue StandardError => error
+          warn "Skipping Keinaufwand asset #{url}: #{error.message}"
+        end
+      end
+
+      def local_record
+        @local_record ||= @record_class.where(type: resource_type).find_by(id: resource_id)
+      end
+
+      def event_id = @payload[:event_id]
+      def event_type = @payload[:event_type].to_s
+      def api_version = @payload[:api_version].to_s
+      def data = @payload[:data]
+      def resource_id = data[:id]
+      def deleted? = data[:_deleted] == true
+
+      def source_updated_at
+        @source_updated_at ||= Time.iso8601(data[:updated_at].to_s) if data[:updated_at].present?
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/webhook_receipt.rb

@@ -0,0 +1,13 @@
+module Keinaufwand
+  module Sync
+    class WebhookReceipt < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name = "keinaufwand_webhook_receipts"
+      self.primary_key = :event_id
+
+      def self.latest_source_updated_at(resource_type, resource_id)
+        where(resource_type: resource_type, resource_id: resource_id).maximum(:source_updated_at)
+      end
+    end
+  end
+end