amplify_syndication 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c904621a09cb44ba73a694784f8f3ca99361b845f4f8f79e9c07d284b32f1f7c
-  data.tar.gz: '01094a0769aa29721d17bf14567daa765debbb580a9456913139459ec25d2bb1'
+  metadata.gz: 6b508d426a6c47cead51ad5c59d84322c749a60dc4155add4454a53a388e38e2
+  data.tar.gz: 83b82be51f0a21440dfb5ab571a28f11bbbd53e7d23de8ca6b9570a0e6aa9813
 SHA512:
-  metadata.gz: 82ac0a3a1682ed77be6487b41be10b41b1f48efb9fed201ee1c5c9935dbd2d965c5e68a199b4103fb1f2ae5f0b2f3838dcdb34532be28536ff27a5e299100449
-  data.tar.gz: 42309083ecdf14c0112dad8d8d36438daf591648ccb3bd374ba5925a8ec8d588a23224fe166b5bd4c59bde993f25eed79b2e033237039f43e3e5c366b7933448
+  metadata.gz: b8ed56083a3c0eab1bcab901a10d8ea6946d0be4c36171387c19a68f23b389580378b6f821629d0197bedd63ddf8c9636b320c97ca85416c59a22795ca00f7fc
+  data.tar.gz: 625a208f80f0e38a23a03f75aa2fe7d86bd7a569ab79a0882e5500c4c8376ac1cf6b3ded8547ca48c677efe03a4fce7cef895212c3f80f31e5f86b8511ba9e95

data/README.md

@@ -153,6 +153,24 @@ puts recent_media
 
 ---
 
+## Lookups
+
+### Fetch all lookups (simple)
+
+Fetch the entire `Lookup` table into memory:
+
+```ruby
+api = AmplifySyndication::API.new
+
+lookups = api.fetch_all_lookups(
+  batch_size: 100,
+  sleep_seconds: 2 # optional throttle between API calls
+)
+
+puts "Loaded #{lookups.size} lookup rows"
+
+---
+
 ## Error Handling
 
 If an API call fails, the gem raises a StandardError with details of the HTTP response:

data/lib/amplify_syndication/api.rb

@@ -1,15 +1,22 @@
+require "uri"
+
 module AmplifySyndication
   class API
     def initialize(client = Client.new)
       @client = client
     end
 
+    # === Metadata ===
+
     # Fetch metadata
     def fetch_metadata
       @client.get("$metadata?$format=json")
     end
 
-    # Fetch property fields
+    # === Field helpers ===
+
+    # Fetch all Field records for the Property resource in a single call
+    # (still paginated under the hood).
     def fetch_property_fields(batch_size: 50, sleep_seconds: 10)
       offset = 0
       fields = []
@@ -17,57 +24,83 @@ module AmplifySyndication
       loop do
         query_options = {
           "$filter" => "ResourceName eq 'Property'",
-          "$top" => batch_size,
-          "$skip" => offset
+          "$top"    => batch_size,
+          "$skip"   => offset
         }.compact
+
         response = fetch_with_options("Field", query_options)
-        batch = response["value"]
+        batch    = response["value"] || []
         break if batch.empty?
 
         fields.concat(batch)
-        
         offset += batch_size
 
-        # Safety sleep between API calls
         sleep(sleep_seconds) if sleep_seconds.positive?
       end
 
       fields
     end
 
-    def fetch_all_lookups(batch_size: 50, sleep_seconds: 10, filter: nil)
-      offset = 0
-      values = []
+    # === Lookup helpers ===
 
-      loop do
-        query_options = {
-          "$top"  => batch_size,
-          "$skip" => offset
-        }
-    
-        # Apply filter only when provided
-        if filter.present?
-          combined_filter =
-            filter.is_a?(Array) ? filter.join(" and ") : filter
-          query_options["$filter"] = combined_filter
-        end
+    # Fetch a single Lookup page (batch) with optional filter.
+    #
+    # filter can be:
+    #   - a String: "LookupStatus eq 'Active'"
+    #   - an Array of strings: ["LookupStatus eq 'Active'", "LookupName eq 'PropertyType'"]
+    def fetch_lookup_batch(skip:, top: 50, filter: nil)
+      query_options = {
+        "$top"  => top,
+        "$skip" => skip
+      }
 
-        response = fetch_with_options("Lookup", query_options)
-        batch = response["value"] || []
-        break if batch.empty?
+      if filter
+        combined_filter =
+          filter.is_a?(Array) ? filter.join(" and ") : filter
+        query_options["$filter"] = combined_filter
+      end
 
-        values.concat(batch)
+      response = fetch_with_options("Lookup", query_options)
+      response["value"] || []
+    end
 
-        offset += batch_size
+    # Iterate over Lookup records in batches, yielding each batch.
+    #
+    # Example:
+    #   api.each_lookup_batch(batch_size: 100, filter: "LookupStatus eq 'Active'") do |batch|
+    #     batch.each { |row| VowLookup.upsert_from_row(row) }
+    #   end
+    def each_lookup_batch(batch_size: 50, sleep_seconds: 10, filter: nil)
+      skip = 0
+
+      loop do
+        batch = fetch_lookup_batch(skip: skip, top: batch_size, filter: filter)
+        break if batch.empty?
 
-        # Safety sleep between API calls
+        yield(batch) if block_given?
+
+        skip += batch_size
         sleep(sleep_seconds) if sleep_seconds.positive?
       end
+    end
 
-      values
+    # Fetch all Lookup rows into memory (simple usage).
+    #
+    # For long-running syncs, prefer each_lookup_batch so your app can
+    # handle persistence/checkpointing per batch.
+    def fetch_all_lookups(batch_size: 50, sleep_seconds: 10, filter: nil)
+      results = []
+
+      each_lookup_batch(batch_size: batch_size,
+                        sleep_seconds: sleep_seconds,
+                        filter: filter) do |batch|
+        results.concat(batch)
+      end
+
+      results
     end
 
-    # get lookup name values
+    # Get all rows for a single LookupName (convenience helper).
     def lookup(lookup_name, batch_size: 50, sleep_seconds: 10)
       offset = 0
       values = []
@@ -75,29 +108,31 @@ module AmplifySyndication
       loop do
         query_options = {
           "$filter" => "LookupName eq '#{lookup_name}'",
-          "$top" => batch_size,
-          "$skip" => offset
+          "$top"    => batch_size,
+          "$skip"   => offset
         }.compact
+
         response = fetch_with_options("Lookup", query_options)
-        batch = response["value"]
+        batch    = response["value"] || []
         break if batch.empty?
 
         values.concat(batch)
         offset += batch_size
 
-        # Safety sleep between API calls
         sleep(sleep_seconds) if sleep_seconds.positive?
       end
 
       values
     end
 
-    # Fetch basic property data
+    # === Property helpers ===
+
+    # Fetch basic property data (simple test helper)
     def fetch_property_data(limit = 1)
       @client.get("Property", "$top" => limit)
     end
 
-    # Fetch data with query options
+    # Fetch data with query options against an arbitrary resource
     def fetch_with_options(resource, query_options = {})
       @client.get_with_options(resource, query_options)
     end
@@ -112,6 +147,7 @@ module AmplifySyndication
         "$skip" => skip,
         "$count" => count
       }.compact
+
       fetch_with_options("Property", query_options)
     end
 
@@ -120,82 +156,142 @@ module AmplifySyndication
       fetch_filtered_properties(count: "true", top: 0)
     end
 
-    ### Replication Methods ###
+    # === Replication helpers ===
+    #
+    # These three methods give you flexible control:
+    #
+    #   - fetch_initial_download_batch  -> single page
+    #   - each_initial_download_batch  -> yields per page
+    #   - perform_initial_download     -> fetch everything into memory
 
-    # Perform initial download for replication
-    def perform_initial_download(
+    # Build and fetch a single replication batch for a resource.
+    def fetch_initial_download_batch(
       resource: "Property",
       batch_size: 100,
       fields: ["ModificationTimestamp", "ListingKey"],
       filter: nil,
-      sleep_seconds: 10,
-      checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 },
+      checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 }
     )
-      puts "Starting initial download..."
-      all_records = [] # Array to collect all records
+      encoded_ts = URI.encode_www_form_component(checkpoint[:last_timestamp])
 
-      loop do
-        puts "Fetching batch with timestamp > #{checkpoint[:last_timestamp]} and key > #{checkpoint[:last_key]}..."
+      # checkpoint filter: everything strictly after the last (timestamp, key) pair
+      checkpoint_filter = "(ModificationTimestamp gt #{encoded_ts}) " \
+                          "or (ModificationTimestamp eq #{encoded_ts} and ListingKey gt '#{checkpoint[:last_key]}')"
 
-        # Build batch filter
-        batch_filter = []
-        batch_filter << "#{filter}" if filter
-        batch_filter << "(ModificationTimestamp gt #{URI.encode_www_form_component(checkpoint[:last_timestamp])})"
-        batch_filter << "or (ModificationTimestamp eq #{URI.encode_www_form_component(checkpoint[:last_timestamp])} and ListingKey gt '#{checkpoint[:last_key]}')"
-        batch_filter = batch_filter.join(" ")
+      conditions = []
+      conditions << "(#{filter})" if filter
+      conditions << "(#{checkpoint_filter})"
 
-        # Query options
-        query_options = {
-          "$select" => fields.join(","),
-          "$filter" => batch_filter,
-          "$orderby" => "ModificationTimestamp,ListingKey",
-          "$top" => batch_size
-        }
+      query_options = {
+        "$select" => fields.join(","),
+        "$filter" => conditions.join(" and "),
+        "$orderby" => "ModificationTimestamp,ListingKey",
+        "$top" => batch_size
+      }
+
+      response = fetch_with_options(resource, query_options)
+      response["value"] || []
+    end
 
-        # Debugging: Print the full query options
-        puts "Query options: #{query_options.inspect}"
+    # Iterate over replication batches, yielding [batch, checkpoint].
+    #
+    # You can persist checkpoint per batch to resume later if something fails.
+    def each_initial_download_batch(
+      resource: "Property",
+      batch_size: 100,
+      fields: ["ModificationTimestamp", "ListingKey"],
+      filter: nil,
+      sleep_seconds: 10,
+      checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 }
+    )
+      loop do
+        batch = fetch_initial_download_batch(
+          resource: resource,
+          batch_size: batch_size,
+          fields: fields,
+          filter: filter,
+          checkpoint: checkpoint
+        )
 
-        # Send request
-        response = fetch_with_options(resource, query_options)
-        records = response["value"]
-        break if records.empty?
+        break if batch.empty?
 
-        # Collect batch records
-        all_records.concat(records)
+        yield(batch, checkpoint) if block_given?
 
-        # Update checkpoint with the last record in the batch
-        last_record = records.last
+        # Update checkpoint automatically based on the last record in the batch
+        last_record = batch.last
         checkpoint[:last_timestamp] = last_record["ModificationTimestamp"]
-        checkpoint[:last_key] = last_record["ListingKey"]
+        checkpoint[:last_key]       = last_record["ListingKey"]
 
-        # Stop if the number of records is less than the batch size
-        break if records.size < batch_size
+        break if batch.size < batch_size
 
-        # Safety sleep between API calls
         sleep(sleep_seconds) if sleep_seconds.positive?
       end
-
-      puts "Initial download complete."
-      all_records # Return the collected records
     end
 
-    # Fetch updates since the last checkpoint
-    def fetch_updates(
-        resource: "Property",
-        batch_size: 100,
-        fields: ["ModificationTimestamp", "ListingKey"],
-        filter: nil,
-        checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 }
-      )
-      perform_initial_download(
+    # Perform initial download for replication, buffering all results
+    # into memory (simple usage).
+    #
+    # For large datasets, prefer each_initial_download_batch.
+    def perform_initial_download(
+      resource: "Property",
+      batch_size: 100,
+      fields: ["ModificationTimestamp", "ListingKey"],
+      filter: nil,
+      sleep_seconds: 10,
+      checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 }
+    )
+      results = []
+
+      puts "Starting initial download..."
+
+      each_initial_download_batch(
         resource: resource,
         batch_size: batch_size,
         fields: fields,
         filter: filter,
+        sleep_seconds: sleep_seconds,
         checkpoint: checkpoint
-      ) do |batch|
-        # Process updates
-        yield(batch) if block_given?
+      ) do |batch, _checkpoint|
+        results.concat(batch)
+      end
+
+      puts "Initial download complete."
+      results
+    end
+
+    # Fetch updates since the last checkpoint.
+    #
+    # If a block is given, yields each batch; otherwise returns all
+    # updates in a single array (same as perform_initial_download).
+    def fetch_updates(
+      resource: "Property",
+      batch_size: 100,
+      fields: ["ModificationTimestamp", "ListingKey"],
+      filter: nil,
+      checkpoint: { last_timestamp: "1970-01-01T00:00:00Z", last_key: 0 },
+      sleep_seconds: 10
+    )
+      if block_given?
+        each_initial_download_batch(
+          resource: resource,
+          batch_size: batch_size,
+          fields: fields,
+          filter: filter,
+          sleep_seconds: sleep_seconds,
+          checkpoint: checkpoint
+        ) do |batch, cp|
+          yield(batch, cp)
+        end
+        nil
+      else
+        perform_initial_download(
+          resource: resource,
+          batch_size: batch_size,
+          fields: fields,
+          filter: filter,
+          sleep_seconds: sleep_seconds,
+          checkpoint: checkpoint
+        )
       end
     end
 
@@ -206,7 +302,7 @@ module AmplifySyndication
       @client.get(endpoint)
     end
 
-    ### Media Methods ###
+    # === Media helpers ===
 
     # Fetch a media record by MediaKey
     def fetch_media_by_key(media_key)
@@ -222,22 +318,25 @@ module AmplifySyndication
       batch_size: 100
     )
       query_options = {
-        "$filter" => "#{filter} and ModificationTimestamp ge #{modification_date}",
+        "$filter"  => "#{filter} and ModificationTimestamp ge #{modification_date}",
         "$orderby" => orderby,
-        "$top" => batch_size
+        "$top"     => batch_size
       }
+
       fetch_with_options("Media", query_options)
     end
 
     # Fetch media by ResourceName and ResourceRecordKey
     def fetch_media_by_resource(resource_name, resource_key, batch_size = 100)
       filter = "ResourceRecordKey eq '#{resource_key}' and ResourceName eq '#{resource_name}'"
+
       query_options = {
-        "$filter" => filter,
+        "$filter"  => filter,
         "$orderby" => "ModificationTimestamp,MediaKey",
-        "$top" => batch_size
+        "$top"     => batch_size
       }
+
       fetch_with_options("Media", query_options)
     end
   end
-end
+end

data/lib/amplify_syndication/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AmplifySyndication
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: amplify_syndication
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Stephen Higgins