keinaufwand-sync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 63f03abe9ebcc27c8e013e60ed0fe51a076a6228e306deeceea89ed14014668b
+  data.tar.gz: 8609e5ea6b0aa0d167efe1b9bc3976b233bcb6b6d84d8543986d590850b3975d
+SHA512:
+  metadata.gz: 6f35691e820d95ddee550dc270a5ea96e6399e3c5568e98db9077ca60abd1c1f0b968b0668ac4601f4a8dafcd207517426aeaead24dd0e293d8069cb867c0b71
+  data.tar.gz: f68b41369ba5e2d2022c04ca11d9ad8e954fc6eb2aa54da76f46feca3ca3add11b39b9f9a98df8dfdcf16e7f92f3fc377b3a1d5834e7cb92714670828895dd71

data/README.md

@@ -0,0 +1,48 @@
+# keinaufwand-sync
+
+Reusable Rails persistence for the Keinaufwand API. The gem supports full/delta API sync, schema-backed Active Record models, SQLite and PostgreSQL JSON queries, local asset mirroring, generated JSON indexes, and signed incremental webhooks.
+
+Part of [Keinaufwand](https://keinaufwand.com).
+
+## Install
+
+```ruby
+gem "keinaufwand"
+gem "keinaufwand-sync"
+```
+
+```sh
+bin/rails generate keinaufwand:sync:install
+bin/rails keinaufwand:indices:generate
+bin/rails db:migrate
+```
+
+The install generator creates `keinaufwand_records`, `keinaufwand_sync_states`, `keinaufwand_assets`, and `keinaufwand_webhook_receipts`. It also mounts `/media/:id` and the webhook Engine at `/webhooks/keinaufwand`.
+
+## Configuration
+
+```ruby
+Keinaufwand::Sync.configure do |config|
+  config.client = -> { MyKeinaufwandClient.build }
+  config.webhook_secret = -> { Rails.application.credentials.dig(:keinaufwand, :webhook_secret) }
+  config.storage_adapter = :sqlite3
+end
+```
+
+The consumer selects resources and JSON indexes in `config/keinaufwand.yml`. The versioned API schema remains bundled in `keinaufwand`.
+
+```sh
+bin/rails keinaufwand:sync
+bin/rails keinaufwand:resync
+bin/rails keinaufwand:indices:generate
+```
+
+Application models inherit from `Keinaufwand::Record`. Their `id` is the upstream ID; `id` and STI `type` form the composite primary key. Schema fields and associations use normal Active Record-style `find`, `exists?`, `where`, and `order` calls.
+
+## Webhooks
+
+The mounted Engine accepts the exact JSON body signed by Keinaufwand in `X-Keinaufwand-Signature`. Processing is synchronous so a `200` response means the SQLite/PostgreSQL transaction committed. Event receipts make retries idempotent and preserve the latest source timestamp so an older delayed update cannot overwrite or resurrect newer data.
+
+Regular `keinaufwand:sync` calls request each aggregate root with `updated_since` and apply `_deleted` records returned by widened soft-visibility scopes. Inline associations are stored inside the root JSON and exposed as schema-backed model objects without duplicate STI rows. Retried webhooks carry hard-delete tombstones; `keinaufwand:resync` repairs exceptional drift after a permanently failed delivery.
+
+Polling compares source timestamps with both local records and webhook receipts after downloading media, then applies each page in a database transaction. Older polling snapshots cannot overwrite a newer webhook update or resurrect a record deleted by a newer webhook. Index generation skips unchanged indexes and uses distinct migration names for later configuration changes.

data/config/routes.rb

@@ -0,0 +1,3 @@
+Keinaufwand::Sync::Engine.routes.draw do
+  post "/", to: "webhooks#receive"
+end

data/lib/generators/keinaufwand/sync/indexes/indexes_generator.rb

@@ -0,0 +1,25 @@
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Keinaufwand
+  module Sync
+    class IndexesGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      def create_index_migration
+        manager = Keinaufwand::Sync.configuration.index_manager
+        operations = manager.migration_operations
+
+        if operations.empty?
+          say "Keinaufwand sync indexes are already up to date."
+          return
+        end
+
+        migration_number = self.class.next_migration_number(File.join(destination_root, "db/migrate"))
+        class_name = "UpdateKeinaufwandSyncIndexes#{migration_number}"
+        source = manager.migration_source(class_name: class_name)
+        create_file File.join("db/migrate", "#{migration_number}_#{class_name.underscore}.rb"), source
+      end
+    end
+  end
+end

data/lib/generators/keinaufwand/sync/install/install_generator.rb

@@ -0,0 +1,35 @@
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Keinaufwand
+  module Sync
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :media_route, type: :boolean, default: true, desc: "Mount /media/:id for synced Keinaufwand assets"
+      class_option :webhook_route, type: :boolean, default: true, desc: "Mount the signed Keinaufwand webhook receiver"
+
+      def copy_migrations
+        migration_template "create_keinaufwand_sync_tables.rb", "db/migrate/create_keinaufwand_sync_tables.rb"
+        migration_template "create_keinaufwand_assets.rb", "db/migrate/create_keinaufwand_assets.rb"
+        migration_template "create_keinaufwand_webhook_receipts.rb", "db/migrate/create_keinaufwand_webhook_receipts.rb"
+      end
+
+      def add_webhook_route
+        return unless options[:webhook_route]
+
+        route %(mount Keinaufwand::Sync::Engine => "/webhooks/keinaufwand", as: :keinaufwand_sync)
+      end
+
+      def add_media_route
+        return unless options[:media_route]
+
+        route %(get "media/:id", to: "keinaufwand/assets#show", as: :keinaufwand_asset)
+      end
+
+
+    end
+  end
+end

data/lib/generators/keinaufwand/sync/install/templates/create_keinaufwand_assets.rb

@@ -0,0 +1,19 @@
+class CreateKeinaufwandAssets < ActiveRecord::Migration<%= "[#{ActiveRecord::Migration.current_version}]" %>
+  def change
+    unless table_exists?(:keinaufwand_assets)
+      create_table :keinaufwand_assets, id: false do |t|
+        t.string :key, null: false
+        t.string :source_url, null: false
+        t.string :content_type, null: false
+        t.integer :byte_size, null: false
+        t.string :checksum, null: false
+        t.binary :data, null: false
+        t.datetime :synced_at, null: false
+        t.timestamps
+      end
+    end
+
+    add_index :keinaufwand_assets, :key, unique: true unless index_exists?(:keinaufwand_assets, :key)
+    add_index :keinaufwand_assets, :source_url, unique: true unless index_exists?(:keinaufwand_assets, :source_url)
+  end
+end

data/lib/generators/keinaufwand/sync/install/templates/create_keinaufwand_sync_tables.rb

@@ -0,0 +1,24 @@
+class CreateKeinaufwandSyncTables < ActiveRecord::Migration<%= "[#{ActiveRecord::Migration.current_version}]" %>
+  def change
+    unless table_exists?(:keinaufwand_records)
+      create_table :keinaufwand_records, primary_key: [:id, :type] do |t|
+        t.bigint :id, null: false
+        t.string :type, null: false
+        t.json :data, null: false, default: {}
+        t.timestamps
+      end
+    end
+
+    add_index :keinaufwand_records, :type unless index_exists?(:keinaufwand_records, :type)
+
+    unless table_exists?(:keinaufwand_sync_states)
+      create_table :keinaufwand_sync_states, id: false do |t|
+        t.string :resource_class, null: false
+        t.datetime :last_synced_at
+        t.timestamps
+      end
+    end
+
+    add_index :keinaufwand_sync_states, :resource_class, unique: true unless index_exists?(:keinaufwand_sync_states, :resource_class)
+  end
+end

data/lib/generators/keinaufwand/sync/install/templates/create_keinaufwand_webhook_receipts.rb

@@ -0,0 +1,16 @@
+class CreateKeinaufwandWebhookReceipts < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :keinaufwand_webhook_receipts, id: false do |t|
+      t.bigint :event_id, null: false, primary_key: true
+      t.string :event_type, null: false
+      t.string :api_version, null: false
+      t.string :resource_type, null: false
+      t.bigint :resource_id, null: false
+      t.datetime :source_updated_at
+      t.string :result, null: false
+      t.datetime :processed_at, null: false
+    end
+
+    add_index :keinaufwand_webhook_receipts, [:resource_type, :resource_id, :source_updated_at], name: "idx_keinaufwand_webhook_receipts_resource"
+  end
+end

data/lib/keinaufwand/asset.rb

@@ -0,0 +1,14 @@
+module Keinaufwand
+  class Asset < Sync::Asset
+    self.abstract_class = false
+    self.primary_key = :key
+
+    def self.local_url_for(source_url)
+      return if source_url.blank?
+      return source_url unless asset_url?(source_url)
+
+      key = key_for(source_url)
+      exists?(key:) ? "#{Sync.configuration.local_asset_path_prefix}/#{key}" : source_url
+    end
+  end
+end

data/lib/keinaufwand/assets_controller.rb

@@ -0,0 +1,12 @@
+module Keinaufwand
+  class AssetsController < ActionController::Base
+    def show
+      asset = Asset.find(params[:id])
+
+      if stale?(etag: asset.checksum, last_modified: asset.synced_at, public: true)
+        expires_in 1.year, public: true
+        send_data asset.data, type: asset.content_type, disposition: "inline"
+      end
+    end
+  end
+end

data/lib/keinaufwand/record.rb

@@ -0,0 +1,32 @@
+module Keinaufwand
+  class Record < Sync::Record
+    self.abstract_class = false
+
+    def value(key)
+      data[key.to_s]
+    end
+
+    def [](key)
+      key = key.to_s
+      return data[key] if data.is_a?(Hash) && data.key?(key)
+
+      super
+    end
+
+    def nested(*keys)
+      keys.reduce(data) { |hash, key| hash.is_a?(Hash) ? hash[key.to_s] : nil }
+    end
+
+    def local_asset_url(source_url)
+      Asset.local_url_for(source_url)
+    end
+
+    def localize_asset_urls(value)
+      return value unless value.is_a?(String)
+
+      Asset.urls_from(value).uniq.reduce(value) do |body, url|
+        body.gsub(url, Asset.local_url_for(url) || url)
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync.rb

@@ -0,0 +1,30 @@
+require "active_record"
+require "action_controller/railtie"
+require "active_support/core_ext/object/blank"
+require "bigdecimal/util"
+require "cgi"
+require "digest"
+require "net/http"
+require "openssl"
+require "uri"
+require "yaml"
+require "active_support/core_ext/numeric/time"
+require "keinaufwand/api"
+
+require_relative "sync_core/version"
+require_relative "sync_core/configuration"
+require_relative "sync_core/asset"
+require_relative "sync_core/record"
+require_relative "sync_core/sync_state"
+require_relative "sync_core/webhook_receipt"
+require_relative "sync_core/sync_service"
+require_relative "sync_core/webhook_processor"
+require_relative "sync_core/webhooks_controller" if defined?(ActionController::API)
+require_relative "sync_core/index_manager"
+require_relative "sync_core/engine" if defined?(Rails::Engine)
+require_relative "sync_core/railtie" if defined?(Rails::Railtie)
+require_relative "record"
+require_relative "asset"
+require_relative "sync_state"
+require_relative "webhook_receipt"
+require_relative "assets_controller" if defined?(ActionController::Base)

data/lib/keinaufwand/sync_core/asset.rb

@@ -0,0 +1,69 @@
+module Keinaufwand
+  module Sync
+    class Asset < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name = "keinaufwand_assets"
+      self.primary_key = :key
+
+      ASSET_URL_PATTERN = %r{\Ahttps?://.+/shrine/(?:production|development)/(?:store|cache)/}.freeze
+      URL_PATTERN = %r{https?://[^"'<>\s\\]+}.freeze
+
+      def self.sync_url(source_url)
+        return unless asset_url?(source_url)
+
+        key = key_for(source_url)
+        return find_by(key:) if exists?(key:)
+
+        response = fetch(source_url)
+        body = response.body.to_s.b
+        content_type = response["content-type"].to_s.split(";").first
+        create!(
+          key: key,
+          source_url: source_url,
+          content_type: content_type.empty? ? "application/octet-stream" : content_type,
+          byte_size: body.bytesize,
+          checksum: Digest::SHA256.hexdigest(body),
+          data: body,
+          synced_at: Time.current
+        )
+      end
+
+      def self.key_for(source_url)
+        Digest::SHA256.hexdigest(source_url.to_s)
+      end
+
+      def self.asset_url?(url)
+        url.to_s.match?(ASSET_URL_PATTERN)
+      end
+
+      def self.urls_from(value, urls = [])
+        case value
+        when Hash
+          value.each_value { |child| urls_from(child, urls) }
+        when Array
+          value.each { |child| urls_from(child, urls) }
+        when String
+          CGI.unescapeHTML(value).scan(URL_PATTERN).each do |url|
+            urls << url if asset_url?(url)
+          end
+        end
+
+        urls
+      end
+
+      def self.fetch(url, limit = 3)
+        raise ArgumentError, "too many redirects for #{url}" if limit < 0
+
+        uri = URI(url)
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+          http.get(uri.request_uri)
+        end
+
+        return fetch(response["location"], limit - 1) if response.is_a?(Net::HTTPRedirection)
+        raise "Could not fetch #{url}: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+        response
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/configuration.rb

@@ -0,0 +1,101 @@
+module Keinaufwand
+  module Sync
+    class Configuration
+      attr_accessor :client, :record_class, :sync_state_class, :asset_class, :webhook_receipt_class, :webhook_secret, :resources, :config_path, :local_asset_path_prefix, :storage_adapter
+
+      def initialize
+        @record_class = -> { Keinaufwand::Record }
+        @sync_state_class = -> { Keinaufwand::SyncState }
+        @asset_class = -> { Keinaufwand::Asset }
+        @webhook_receipt_class = -> { Keinaufwand::WebhookReceipt }
+        @local_asset_path_prefix = "/media"
+      end
+
+      def sync_service
+        SyncService.new(
+          client: fetch(:client),
+          record_class: fetch(:record_class),
+          sync_state_class: fetch(:sync_state_class),
+          receipt_class: resolve(:webhook_receipt_class),
+          resources: resources || sync_config.map { |item| item.fetch("model") }.presence || SyncService::DEFAULT_RESOURCES,
+          asset_class: resolve(:asset_class)
+        )
+      end
+
+      def sync_config
+        return @sync_config if defined?(@sync_config)
+
+        @sync_config = Array(app_config.dig(:sync, :resources))
+      end
+
+      def index_manager
+        IndexManager.new(sync_config: sync_config, adapter: storage_adapter || :sqlite3)
+      end
+
+      def webhook_processor(payload)
+        WebhookProcessor.new(
+          payload: payload,
+          record_class: fetch(:record_class),
+          receipt_class: fetch(:webhook_receipt_class),
+          resources: resources || sync_config.map { |item| item.fetch("model") }.presence || SyncService::DEFAULT_RESOURCES,
+          asset_class: resolve(:asset_class)
+        )
+      end
+
+      def resolved_webhook_secret
+        fetch(:webhook_secret)
+      end
+
+      def app_config
+        return {}.with_indifferent_access unless resolved_config_path && File.exist?(resolved_config_path)
+
+        YAML.load_file(resolved_config_path).with_indifferent_access
+      end
+
+      def clear!
+        resolve(:asset_class)&.delete_all
+        fetch(:record_class).delete_all
+        fetch(:sync_state_class).delete_all
+        resolve(:webhook_receipt_class)&.delete_all
+      end
+
+      private
+
+      def fetch(name)
+        value = resolve(name)
+        raise ArgumentError, "Keinaufwand::Sync.#{name} is not configured" unless value
+
+        value
+      end
+
+      def resolve(name)
+        value = public_send(name)
+        value = value.call if value.respond_to?(:call)
+        value
+      end
+
+      def resolved_config_path
+        resolve(:config_path) || (Rails.root.join("config/keinaufwand.yml") if defined?(Rails))
+      end
+    end
+
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield configuration
+      end
+
+      def sync!
+        configuration.sync_service.perform
+      end
+
+      def resync!
+        configuration.clear!
+        sync!
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/engine.rb

@@ -0,0 +1,7 @@
+module Keinaufwand
+  module Sync
+    class Engine < Rails::Engine
+      isolate_namespace Keinaufwand::Sync
+    end
+  end
+end

data/lib/keinaufwand/sync_core/index_manager.rb

@@ -0,0 +1,83 @@
+module Keinaufwand
+  module Sync
+    class IndexManager
+      TABLE_NAME = "keinaufwand_records"
+
+      def initialize(sync_config: [], adapter: :sqlite3)
+        @sync_config = sync_config
+        @adapter = adapter.to_sym
+      end
+
+      def desired_indices
+        @sync_config.each_with_object({}) do |item, indices|
+          model = item.fetch("model")
+          Array(item["indices"]).each do |attribute|
+            indices.merge!(build_index(model, attribute))
+          end
+        end
+      end
+
+      def current_index_names(connection = ActiveRecord::Base.connection)
+        return [] unless connection.table_exists?(TABLE_NAME)
+
+        connection.indexes(TABLE_NAME).map(&:name).grep(/\Aidx_keinaufwand_/)
+      end
+
+      def missing_index_names(connection = ActiveRecord::Base.connection)
+        desired_indices.keys - current_index_names(connection)
+      end
+
+      def obsolete_index_names(connection = ActiveRecord::Base.connection)
+        current_index_names(connection) - desired_indices.keys
+      end
+
+      def migration_source(class_name: "UpdateKeinaufwandSyncIndexes", mode: :diff)
+        operations = migration_operations(mode: mode)
+
+        <<~RUBY
+          class #{class_name} < ActiveRecord::Migration[#{ActiveRecord::Migration.current_version}]
+            def change
+          #{operations.join("\n")}
+            end
+          end
+        RUBY
+      end
+
+      def migration_operations(connection = ActiveRecord::Base.connection, mode: :diff)
+        desired = desired_indices
+        removals = obsolete_index_names(connection).map do |name|
+          "    remove_index :#{TABLE_NAME}, name: #{name.inspect} if index_name_exists?(:#{TABLE_NAME}, #{name.inspect})"
+        end
+        addition_names = mode.to_sym == :full ? desired.keys : missing_index_names(connection)
+        additions = addition_names.map do |name|
+          config = desired.fetch(name)
+          "    add_index :#{TABLE_NAME}, #{config[:expression].inspect}, name: #{name.inspect}, where: #{config[:where].inspect} unless index_name_exists?(:#{TABLE_NAME}, #{name.inspect})"
+        end
+
+        removals + additions
+      end
+
+      private
+
+      def build_index(model, attribute)
+        attribute = attribute.to_s
+        name = "idx_keinaufwand_#{model.underscore}_#{attribute.tr("^a-zA-Z0-9_", "_")}"
+
+        {
+          name => {
+            expression: json_value_expression(attribute),
+            where: "type = '#{model}'"
+          }
+        }
+      end
+
+      def json_value_expression(attribute)
+        if @adapter == :postgres || @adapter == :postgresql
+          "data #>> '{#{attribute.split(".").join(",")}}'"
+        else
+          "json_extract(data, '$.#{attribute}')"
+        end
+      end
+    end
+  end
+end

data/lib/keinaufwand/sync_core/railtie.rb

@@ -0,0 +1,9 @@
+module Keinaufwand
+  module Sync
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load File.expand_path("tasks/keinaufwand_sync.rake", __dir__)
+      end
+    end
+  end
+end