keinaufwand-sync 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63f03abe9ebcc27c8e013e60ed0fe51a076a6228e306deeceea89ed14014668b
-  data.tar.gz: 8609e5ea6b0aa0d167efe1b9bc3976b233bcb6b6d84d8543986d590850b3975d
+  metadata.gz: 99d27c728ca508889f1696b869941416fce9bc1edccda9b808b9be1c1945be23
+  data.tar.gz: d2c70f51bbbfb3f53baeaa0b18c56b23b95914ec93bbe3661ca9bb15424c80b1
 SHA512:
-  metadata.gz: 6f35691e820d95ddee550dc270a5ea96e6399e3c5568e98db9077ca60abd1c1f0b968b0668ac4601f4a8dafcd207517426aeaead24dd0e293d8069cb867c0b71
-  data.tar.gz: f68b41369ba5e2d2022c04ca11d9ad8e954fc6eb2aa54da76f46feca3ca3add11b39b9f9a98df8dfdcf16e7f92f3fc377b3a1d5834e7cb92714670828895dd71
+  metadata.gz: e6fbe5748850a2abb6f6ac2f672afe2ecf7e02400b33ade8b2069fe7bc1602e48dce607bca39d50a70774fcd90bde9fca6c1a0a5740a76a42775dedabf908b96
+  data.tar.gz: 2f7a1e3e3244914b78e0039167d23db5d835717ba2baac1866a3618d7d93abb8b159da78a593d526249b6fc7944af6e5bbb800317124e3607bdde6a84084af50

data/README.md

@@ -39,6 +39,20 @@ bin/rails keinaufwand:indices:generate
 
 Application models inherit from `Keinaufwand::Record`. Their `id` is the upstream ID; `id` and STI `type` form the composite primary key. Schema fields and associations use normal Active Record-style `find`, `exists?`, `where`, and `order` calls.
 
+## Console progress
+
+Version 0.1.1 adds optional console output without changing the default quiet sync:
+
+```ruby
+Keinaufwand::Sync.sync!(output: $stdout)
+```
+
+Output includes the effective sync cursor, committed per-page New/Upd/Del/Skip counts, the API record total, percentage, records per second, estimated time remaining for the current resource, per-resource fetch/media/database-write timings, and a final completion summary. Lines are flushed immediately, including when stdout is redirected. `Upd` counts existing records written by the sync, even if their data was unchanged. `Skip` counts stale snapshots and deletion markers for records already absent. Counts describe this polling run; concurrent webhook writes are not included.
+
+Progress updates after each committed page (up to 100 records). ETA uses the current resource’s average processing rate, including fetch and media time; it is approximate and shown as unknown when the API omits its total. Page fetch and media-processing messages identify work between updates.
+
+Failed resources report the error and stop the run with the original exception. Rolled-back pages are not counted as processed, and the resource cursor only advances after the complete resource sync succeeds.
+
 ## Webhooks
 
 The mounted Engine accepts the exact JSON body signed by Keinaufwand in `X-Keinaufwand-Signature`. Processing is synchronous so a `200` response means the SQLite/PostgreSQL transaction committed. Event receipts make retries idempotent and preserve the latest source timestamp so an older delayed update cannot overwrite or resurrect newer data.

data/lib/keinaufwand/sync_core/configuration.rb

@@ -11,14 +11,15 @@ module Keinaufwand
         @local_asset_path_prefix = "/media"
       end
 
-      def sync_service
+      def sync_service(output: nil)
         SyncService.new(
           client: fetch(:client),
           record_class: fetch(:record_class),
           sync_state_class: fetch(:sync_state_class),
           receipt_class: resolve(:webhook_receipt_class),
           resources: resources || sync_config.map { |item| item.fetch("model") }.presence || SyncService::DEFAULT_RESOURCES,
-          asset_class: resolve(:asset_class)
+          asset_class: resolve(:asset_class),
+          output: output
         )
       end
 
@@ -88,8 +89,8 @@ module Keinaufwand
         yield configuration
       end
 
-      def sync!
-        configuration.sync_service.perform
+      def sync!(output: nil)
+        configuration.sync_service(output: output).perform
       end
 
       def resync!

data/lib/keinaufwand/sync_core/sync_service.rb

@@ -5,53 +5,112 @@ module Keinaufwand
         Location Product Session EventDetail Event Post InstructorGroup
       ].freeze
 
-      def initialize(client:, record_class:, sync_state_class:, resources: DEFAULT_RESOURCES, asset_class: nil, receipt_class: nil)
+      def initialize(client:, record_class:, sync_state_class:, resources: DEFAULT_RESOURCES, asset_class: nil, receipt_class: nil, output: nil)
         @client = client
         @record_class = record_class
         @sync_state_class = sync_state_class
         @resources = resources
         @asset_class = asset_class
         @receipt_class = receipt_class
+        @output = output
       end
 
       def perform
-        @resources.each { |resource| sync_resource(resource) }
+        started = monotonic_time
+        succeeded = 0
+        log "Starting Keinaufwand Sync (#{@resources.size} resources)"
+        @resources.each_with_index do |resource, index|
+          begin
+            sync_resource(resource, index + 1)
+            succeeded += 1
+          rescue StandardError => error
+            log "!! FAILED #{resource}: #{error.message}"
+            log "Sync stopped (#{succeeded}/#{@resources.size} succeeded)"
+            raise
+          end
+        end
+        log "Sync Process Completed in #{format('%.2f', monotonic_time - started)}s (#{succeeded}/#{@resources.size} succeeded)"
       end
 
       private
 
-      def sync_resource(resource)
+      def sync_resource(resource, position)
+        started = monotonic_time
+        stats = { created: 0, updated: 0, destroyed: 0, skipped: 0 }
+        processed = 0
+        pages = 0
+        timings = { fetch: 0.0, media: 0.0, write: 0.0 }
         cursor = @sync_state_class.cursor_for(resource)
         params = { limit: 100 }
         params[:updated_since] = (cursor - 10.seconds).iso8601 if cursor
+        log "[#{position}/#{@resources.size}] #{resource}: syncing since #{params[:updated_since] || 'beginning'}..."
 
+        fetch_started = monotonic_time
         page = @client.public_send("retrieve_#{resource.pluralize.underscore}", params)
+        timings[:fetch] += monotonic_time - fetch_started
+        total = page.meta.dig("pagination", "count") if page.respond_to?(:meta)
         latest = nil
 
         while page && !page.empty?
+          log "    Page #{pages + 1}: syncing #{page.count} records#{' and media' if @asset_class}..."
+          media_started = monotonic_time
           page.each do |record|
             sync_assets(record.raw_attributes) if @asset_class && record.raw_attributes[:_deleted] != true
           end
+          timings[:media] += monotonic_time - media_started
 
+          write_started = monotonic_time
+          page_stats = { created: 0, updated: 0, destroyed: 0, skipped: 0 }
           @record_class.transaction do
+            existing_ids = @output ? @record_class.where(type: resource, id: page.map(&:id)).pluck(:id) : []
             rows = page.filter_map do |record|
               latest = latest_timestamp(latest, record.server_updated_at)
-              next if stale?(resource, record)
+              if stale?(resource, record)
+                page_stats[:skipped] += 1
+                next
+              end
 
               if record.raw_attributes[:_deleted] == true
-                delete_record(resource, record.id)
+                deleted = delete_record(resource, record.id)
+                page_stats[deleted.positive? ? :destroyed : :skipped] += 1
                 next
               end
 
+              page_stats[existing_ids.include?(record.id) ? :updated : :created] += 1
               { id: record.id, type: resource, data: record.raw_attributes.except(:id), created_at: Time.current, updated_at: Time.current }
             end
 
             @record_class.upsert_all(rows, unique_by: %i[id type], update_only: %w[data updated_at]) if rows.any?
           end
+          timings[:write] += monotonic_time - write_started
+          stats.merge!(page_stats) { |_key, total, count| total + count }
+          pages += 1
+          processed += page.count
+          rate = processed / [monotonic_time - started, 0.001].max
+          progress = total && total.positive? ? "#{processed}/#{total} (#{[processed * 100 / total, 100].min}%)" : processed.to_s
+          eta = total && total.positive? ? "#{format('%.0f', [total - processed, 0].max / rate)}s" : "unknown"
+          log "    Processed: #{progress} (New: #{stats[:created]} | Upd: #{stats[:updated]} | Del: #{stats[:destroyed]} | Skip: #{stats[:skipped]}) [#{format('%.1f', rate)} records/s | ETA: #{eta}]"
+
+          log "    Fetching next page..." if page.respond_to?(:meta) && page.meta.dig("pagination", "next")
+          fetch_started = monotonic_time
           page = page.next
+          timings[:fetch] += monotonic_time - fetch_started
         end
 
         @sync_state_class.update_cursor!(resource, latest) if latest
+        elapsed = monotonic_time - started
+        log "    Sync Report: #{resource} (#{pages} pages, #{processed} records in #{format('%.1f', elapsed)}s)"
+        log "    Fetch: #{format('%.2f', timings[:fetch])}s | Media: #{format('%.2f', timings[:media])}s | Write: #{format('%.2f', timings[:write])}s"
+        log "    Overall: #{format('%.1f', processed / [elapsed, 0.001].max)} records/s"
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def log(message)
+        @output&.puts(message)
+        @output&.flush
       end
 
       def delete_record(resource, id)

data/lib/keinaufwand/sync_core/version.rb

@@ -1,5 +1,5 @@
 module Keinaufwand
   module Sync
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: keinaufwand-sync
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Codegestalt