keinaufwand-sync 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/keinaufwand/sync_core/sync_service.rb +18 -1
- data/lib/keinaufwand/sync_core/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d86399d2bca834ab39cb73597a2be6b7cdc43acb350c5a32e0fa910d81224be3
|
|
4
|
+
data.tar.gz: e11f57515ffac42379c91d5466ff8607b64c269ae547e18c58bff21d4493ddaa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7caab88fabe6cf0cc58dbecd7e9b686d6260bbca4cb1f1734cc006fe2dac434e5779b3825d5847a5e4b20833d8eab499ec786a8de01af9afe05a674063da9c23
|
|
7
|
+
data.tar.gz: a6032bd4f65102d3eba7f8780070e5a01e4f78ba3cc706188abda07532ca89c46834633a5173776ef5ca792c627869045c1e663bebee723378c154c3a8226323
|
data/README.md
CHANGED
|
@@ -57,6 +57,8 @@ Failed resources report the error and stop the run with the original exception.
|
|
|
57
57
|
|
|
58
58
|
The mounted Engine accepts the exact JSON body signed by Keinaufwand in `X-Keinaufwand-Signature`. Processing is synchronous so a `200` response means the SQLite/PostgreSQL transaction committed. Event receipts make retries idempotent and preserve the latest source timestamp so an older delayed update cannot overwrite or resurrect newer data.
|
|
59
59
|
|
|
60
|
-
Regular `keinaufwand:sync` calls request each aggregate root with `updated_since` and
|
|
60
|
+
Regular `keinaufwand:sync` calls request each aggregate root with `updated_since` and delete the records that come back as `_deleted` tombstones. The API lists archived, unpublished and cancelled records with their flags, so the application filters what it shows; a tombstone only means the record was deleted. Inline associations are stored inside the root JSON and exposed as schema-backed model objects without duplicate STI rows.
|
|
61
|
+
|
|
62
|
+
The API keeps deletions for 90 days. When a cursor is older than that, it answers `410 FULL_SYNC_REQUIRED` (raised by the client as `Keinaufwand::Api::Client::FullSyncRequired`), and the sync lists that resource again from the beginning. After a full listing, from a first sync, `keinaufwand:resync` or a `410`, and only once every page succeeded, the sync deletes the local records of that resource the listing didn't contain, except those written after the listing started (by webhooks).
|
|
61
63
|
|
|
62
64
|
Polling compares source timestamps with both local records and webhook receipts after downloading media, then applies each page in a database transaction. Older polling snapshots cannot overwrite a newer webhook update or resurrect a record deleted by a newer webhook. Index generation skips unchanged indexes and uses distinct migration names for later configuration changes.
|
|
@@ -32,13 +32,24 @@ module Keinaufwand
|
|
|
32
32
|
|
|
33
33
|
private
|
|
34
34
|
|
|
35
|
+
# A delta from the cursor, or everything when there is none. When the API no longer has the deletions since the
|
|
36
|
+
# cursor (410 FULL_SYNC_REQUIRED), the resource is listed again from the beginning.
|
|
35
37
|
def sync_resource(resource, position)
|
|
38
|
+
cursor = @sync_state_class.cursor_for(resource)
|
|
39
|
+
sync_listing(resource, position, cursor)
|
|
40
|
+
rescue Keinaufwand::Api::Client::FullSyncRequired => e
|
|
41
|
+
log " #{resource}: #{e.message}"
|
|
42
|
+
sync_listing(resource, position, nil)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def sync_listing(resource, position, cursor)
|
|
36
46
|
started = monotonic_time
|
|
47
|
+
started_at = Time.current
|
|
37
48
|
stats = { created: 0, updated: 0, destroyed: 0, skipped: 0 }
|
|
38
49
|
processed = 0
|
|
39
50
|
pages = 0
|
|
51
|
+
listed_ids = []
|
|
40
52
|
timings = { fetch: 0.0, media: 0.0, write: 0.0 }
|
|
41
|
-
cursor = @sync_state_class.cursor_for(resource)
|
|
42
53
|
params = { limit: 100 }
|
|
43
54
|
params[:updated_since] = (cursor - 10.seconds).iso8601 if cursor
|
|
44
55
|
log "[#{position}/#{@resources.size}] #{resource}: syncing since #{params[:updated_since] || 'beginning'}..."
|
|
@@ -62,6 +73,7 @@ module Keinaufwand
|
|
|
62
73
|
@record_class.transaction do
|
|
63
74
|
existing_ids = @output ? @record_class.where(type: resource, id: page.map(&:id)).pluck(:id) : []
|
|
64
75
|
rows = page.filter_map do |record|
|
|
76
|
+
listed_ids << record.id
|
|
65
77
|
latest = latest_timestamp(latest, record.server_updated_at)
|
|
66
78
|
if stale?(resource, record)
|
|
67
79
|
page_stats[:skipped] += 1
|
|
@@ -95,6 +107,11 @@ module Keinaufwand
|
|
|
95
107
|
timings[:fetch] += monotonic_time - fetch_started
|
|
96
108
|
end
|
|
97
109
|
|
|
110
|
+
# A full listing has every record, so those it didn't list are gone; records webhooks wrote meanwhile stay.
|
|
111
|
+
unless cursor
|
|
112
|
+
swept = @record_class.where(type: resource).where.not(id: listed_ids).where(updated_at: ...started_at).delete_all
|
|
113
|
+
log " Removed #{swept} records the full listing no longer has" if swept.positive?
|
|
114
|
+
end
|
|
98
115
|
@sync_state_class.update_cursor!(resource, latest) if latest
|
|
99
116
|
elapsed = monotonic_time - started
|
|
100
117
|
log " Sync Report: #{resource} (#{pages} pages, #{processed} records in #{format('%.1f', elapsed)}s)"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: keinaufwand-sync
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Codegestalt
|
|
@@ -43,14 +43,14 @@ dependencies:
|
|
|
43
43
|
requirements:
|
|
44
44
|
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: 0.3.
|
|
46
|
+
version: 0.3.1
|
|
47
47
|
type: :runtime
|
|
48
48
|
prerelease: false
|
|
49
49
|
version_requirements: !ruby/object:Gem::Requirement
|
|
50
50
|
requirements:
|
|
51
51
|
- - ">="
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: 0.3.
|
|
53
|
+
version: 0.3.1
|
|
54
54
|
- !ruby/object:Gem::Dependency
|
|
55
55
|
name: railties
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|