feedx 0.12.7 → 0.14.0

data/incremental.go

@@ -0,0 +1,122 @@
+package feedx
+
+import (
+	"context"
+	"errors"
+
+	"github.com/bsm/bfs"
+)
+
+// IncrmentalProduceFunc returns a ProduceFunc closure around an incremental version.
+type IncrementalProduceFunc func(remoteVersion int64) ProduceFunc
+
+// IncrementalProducer pushes incremental feeds to a remote bucket location.
+type IncrementalProducer struct {
+	bucket    bfs.Bucket
+	object    *bfs.Object
+	ownBucket bool
+}
+
+// NewIncrementalProducer inits a new incremental feed producer.
+func NewIncrementalProducer(ctx context.Context, bucketURL string) (*IncrementalProducer, error) {
+	bucket, err := bfs.Connect(ctx, bucketURL)
+	if err != nil {
+		return nil, err
+	}
+
+	pcr := NewIncrementalProducerForBucket(bucket)
+	pcr.ownBucket = true
+	return pcr, nil
+}
+
+// NewIncrementalProducerForRemote starts a new incremental feed producer for a bucket.
+func NewIncrementalProducerForBucket(bucket bfs.Bucket) *IncrementalProducer {
+	return &IncrementalProducer{
+		bucket: bucket,
+		object: bfs.NewObjectFromBucket(bucket, "manifest.json"),
+	}
+}
+
+// Close stops the producer.
+func (p *IncrementalProducer) Close() (err error) {
+	if e := p.object.Close(); e != nil {
+		err = errors.Join(err, e)
+	}
+
+	if p.ownBucket && p.bucket != nil {
+		if e := p.bucket.Close(); e != nil {
+			err = errors.Join(err, e)
+		}
+		p.bucket = nil
+	}
+	return
+}
+
+func (p *IncrementalProducer) Produce(ctx context.Context, version int64, opt *WriterOptions, pfn IncrementalProduceFunc) (*Status, error) {
+	status := Status{LocalVersion: version}
+
+	// fetch manifest from remote object
+	mft, err := loadManifest(ctx, p.object)
+	if err != nil {
+		return nil, err
+	}
+
+	// skip if not modified
+	remoteVersion := mft.Version
+	status.RemoteVersion = remoteVersion
+	if skipSync(version, remoteVersion) {
+		status.Skipped = true
+		return &status, nil
+	}
+
+	// set version for writer
+	if opt == nil {
+		opt = new(WriterOptions)
+	}
+	opt.Version = version
+
+	// write data modified since last version
+	numWritten, err := p.writeDataFile(ctx, mft, version, remoteVersion, opt, pfn)
+	if err != nil {
+		return nil, err
+	}
+	// write new manifest to remote
+	if err := p.commitManifest(ctx, mft, &WriterOptions{Version: version}); err != nil {
+		return nil, err
+	}
+
+	status.NumItems = numWritten
+	return &status, nil
+}
+
+func (p *IncrementalProducer) writeDataFile(ctx context.Context, mft *manifest, version, remoteVersion int64, opt *WriterOptions, pfn IncrementalProduceFunc) (int64, error) {
+	fname := mft.newDataFileName(opt)
+
+	obj := bfs.NewObjectFromBucket(p.bucket, fname)
+	defer obj.Close()
+
+	writer := NewWriter(ctx, obj, opt)
+	defer writer.Discard()
+
+	if err := pfn(remoteVersion)(writer); err != nil {
+		return 0, err
+	}
+	if err := writer.Commit(); err != nil {
+		return 0, err
+	}
+
+	mft.Files = append(mft.Files, fname)
+	mft.Version = version
+
+	return writer.NumWritten(), nil
+}
+
+func (p *IncrementalProducer) commitManifest(ctx context.Context, mft *manifest, opt *WriterOptions) error {
+	writer := NewWriter(ctx, p.object, opt)
+	defer writer.Discard()
+
+	if err := writer.Encode(mft); err != nil {
+		return err
+	}
+	return writer.Commit()
+}

data/incremental_test.go

@@ -0,0 +1,62 @@
+package feedx_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+)
+
+func TestIncrementalProducer(t *testing.T) {
+	bucket := bfs.NewInMem()
+	defer bucket.Close()
+
+	pcr := feedx.NewIncrementalProducerForBucket(bucket)
+	defer pcr.Close()
+
+	// first produce
+	testIncProduce(t, pcr, 101, &feedx.Status{LocalVersion: 101, NumItems: 10})
+
+	// second produce
+	testIncProduce(t, pcr, 101, &feedx.Status{Skipped: true, LocalVersion: 101, RemoteVersion: 101})
+
+	// increment version
+	testIncProduce(t, pcr, 134, &feedx.Status{LocalVersion: 134, RemoteVersion: 101, NumItems: 3})
+
+	obj := bfs.NewObjectFromBucket(bucket, "manifest.json")
+	defer obj.Close()
+
+	mft, err := feedx.LoadManifest(t.Context(), obj)
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	} else if exp := (&feedx.Manifest{
+		Version: 134,
+		Files:   []string{"data-0-101.json", "data-0-134.json"},
+	}); !reflect.DeepEqual(exp, mft) {
+		t.Errorf("expected %#v, got %#v", exp, mft)
+	}
+}
+
+func testIncProduce(t *testing.T, pcr *feedx.IncrementalProducer, version int64, exp *feedx.Status) {
+	t.Helper()
+
+	status, err := pcr.Produce(t.Context(), version, nil, func(sinceVersion int64) feedx.ProduceFunc {
+		return func(w *feedx.Writer) error {
+			n := (version - sinceVersion) / 10
+			for i := int64(0); i < n; i++ {
+				if err := w.Encode(seed()); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	})
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	if !reflect.DeepEqual(exp, status) {
+		t.Errorf("expected %#v, got %#v", exp, status)
+	}
+}

data/lib/feedx/cache/abstract.rb

@@ -17,12 +17,12 @@ class Feedx::Cache::Abstract
   # Fetches data from the cache, using the given key.
   # The optional block will be evaluated and the result stored in the cache
   # in the event of a cache miss.
-  def fetch(key, **opts)
-    value = read(key, **opts)
+  def fetch(key, **)
+    value = read(key, **)
 
     if block_given?
       value ||= yield
-      write(key, value, **opts) if value
+      write(key, value, **) if value
     end
 
     value

data/lib/feedx/cache/value.rb

@@ -8,18 +8,18 @@ class Feedx::Cache::Value
   end
 
   # Read the key.
-  def read(**opts)
-    @cache.read(@key, **opts)
+  def read(**)
+    @cache.read(@key, **)
   end
 
   # Write a value.
-  def write(value, **opts)
-    @cache.write(@key, value, **opts)
+  def write(value, **)
+    @cache.write(@key, value, **)
   end
 
   # Fetches data. The optional block will be evaluated and the
   # result stored in the cache under the key in the event of a cache miss.
-  def fetch(**opts, &block)
-    @cache.fetch(@key, **opts, &block)
+  def fetch(**, &)
+    @cache.fetch(@key, **, &)
   end
 end

data/lib/feedx/compression/abstract.rb

@@ -1,9 +1,9 @@
 class Feedx::Compression::Abstract
-  def reader(_io, **, &_block)
+  def reader(_io, **, &)
     raise 'Not implemented'
   end
 
-  def writer(_io, **, &_block)
+  def writer(_io, **, &)
     raise 'Not implemented'
   end
 end

data/lib/feedx/compression/gzip.rb

@@ -1,14 +1,14 @@
 require 'zlib'
 
 class Feedx::Compression::Gzip < Feedx::Compression::Abstract
-  def reader(io, **, &block)
+  def reader(io, **, &)
     force_binmode(io)
-    Zlib::GzipReader.wrap(io, &block)
+    Zlib::GzipReader.wrap(io, &)
   end
 
-  def writer(io, **, &block)
+  def writer(io, **, &)
     force_binmode(io)
-    Zlib::GzipWriter.wrap(io, &block)
+    Zlib::GzipWriter.wrap(io, &)
   end
 
   private

data/lib/feedx/consumer.rb

@@ -8,8 +8,8 @@ module Feedx
     include Enumerable
 
     # See constructor.
-    def self.each(url, klass, **opts, &block)
-      new(url, klass, **opts).each(&block)
+    def self.each(url, klass, **, &)
+      new(url, klass, **).each(&)
     end
 
     # @param [String] url the destination URL.
@@ -17,7 +17,7 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
+    # @option opts [Feedx::Cache::Value] :cache cache value to store remote version and consume conditionally.
     def initialize(url, klass, format_options: {}, cache: nil, **opts)
       @klass = klass
       @url = url
@@ -32,19 +32,19 @@ module Feedx
     # @return [Boolean] returns true if performed.
     def each(&block)
       stream = Feedx::Stream.new(@url, **@opts)
-      remote_rev = nil
+      remote_ver = nil
 
       if @cache
         metadata   = stream.blob.info.metadata
-        local_rev  = @cache.read.to_i
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return false if remote_rev.positive? && remote_rev <= local_rev
+        local_ver  = @cache.read.to_i
+        remote_ver = (metadata[META_VERSION] || metadata[META_VERSION_DC]).to_i
+        return false if remote_ver.positive? && remote_ver <= local_ver
       end
 
       stream.open do |fmt|
         fmt.decode_each(@klass, **@opts, &block)
       end
-      @cache.write(remote_rev) if @cache && remote_rev
+      @cache.write(remote_ver) if @cache && remote_ver
 
       true
     ensure

data/lib/feedx/format/abstract.rb

@@ -1,15 +1,15 @@
 class Feedx::Format::Abstract
-  def decoder(io, **opts, &block)
-    self.class::Decoder.open(io, **opts, &block)
+  def decoder(io, **, &)
+    self.class::Decoder.open(io, **, &)
   end
 
-  def encoder(io, **opts, &block)
-    self.class::Encoder.open(io, **opts, &block)
+  def encoder(io, **, &)
+    self.class::Encoder.open(io, **, &)
   end
 
   class Wrapper
-    def self.open(io, **opts)
-      inst = new(io, **opts)
+    def self.open(io, **)
+      inst = new(io, **)
       yield inst
     ensure
       inst&.close

data/lib/feedx/format/json.rb

@@ -13,8 +13,8 @@ class Feedx::Format::JSON < Feedx::Format::Abstract
   end
 
   class Encoder < Feedx::Format::Abstract::Encoder
-    def encode(msg, **opts)
-      @io.write msg.to_json(**opts) << "\n"
+    def encode(msg, **)
+      @io.write msg.to_json(**) << "\n"
     end
   end
 end

data/lib/feedx/format/protobuf.rb

@@ -2,8 +2,8 @@ require 'pbio'
 
 class Feedx::Format::Protobuf < Feedx::Format::Abstract
   class Decoder < Feedx::Format::Abstract::Decoder
-    def initialize(io, **opts)
-      super PBIO::Delimited.new(io), **opts
+    def initialize(io, **)
+      super(PBIO::Delimited.new(io), **)
     end
 
     def decode(target, **)
@@ -12,12 +12,12 @@ class Feedx::Format::Protobuf < Feedx::Format::Abstract
   end
 
   class Encoder < Feedx::Format::Abstract::Encoder
-    def initialize(io, **opts)
-      super PBIO::Delimited.new(io), **opts
+    def initialize(io, **)
+      super(PBIO::Delimited.new(io), **)
     end
 
-    def encode(msg, **opts)
-      msg = msg.to_pb(**opts) if msg.respond_to?(:to_pb)
+    def encode(msg, **)
+      msg = msg.to_pb(**) if msg.respond_to?(:to_pb)
       @io.write msg
     end
   end

data/lib/feedx/format.rb

@@ -2,7 +2,6 @@ module Feedx
   module Format
     autoload :Abstract, 'feedx/format/abstract'
     autoload :JSON, 'feedx/format/json'
-    autoload :Parquet, 'feedx/format/parquet'
     autoload :Protobuf, 'feedx/format/protobuf'
 
     class << self
@@ -30,7 +29,7 @@ module Feedx
           kind = _resolve(ext[1..]) || _resolve(ext[1..-2])
           return kind if kind
 
-          base = base[0..-ext.size - 1]
+          base = base[0..(-ext.size - 1)]
         end
       end
 
@@ -41,7 +40,6 @@ module Feedx
           'json'     => :JSON,
           'jsonl'    => :JSON,
           'ndjson'   => :JSON,
-          'parquet'  => :Parquet,
           'pb'       => :Protobuf,
           'proto'    => :Protobuf,
           'protobuf' => :Protobuf,

data/lib/feedx/producer.rb

@@ -6,8 +6,8 @@ module Feedx
   # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
-    def self.perform(url, **opts, &block)
-      new(url, **opts, &block).perform
+    def self.perform(url, **, &)
+      new(url, **, &).perform
     end
 
     # @param [String] url the destination URL.
@@ -15,16 +15,16 @@ module Feedx
     # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
+    # @option opts [Integer] :version the most recent version, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
+    def initialize(url, version: nil, format_options: {}, enum: nil, **opts, &block)
       @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
       @url  = url
       @opts = opts.merge(format_options)
-      @last_mod = last_modified
+      @version = version
 
       return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
 
@@ -34,18 +34,18 @@ module Feedx
     def perform
       Feedx::Stream.open(@url, **@opts) do |stream|
         enum = @enum.is_a?(Proc) ? @enum.call : @enum
-        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+        local_ver = @version.is_a?(Proc) ? @version.call(enum) : @version
+        local_ver = local_ver.to_i
 
         begin
           metadata   = stream.blob.info.metadata
-          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-          return -1 unless local_rev > remote_rev
+          remote_ver = (metadata[META_VERSION] || metadata[META_VERSION_DC]).to_i
+          return -1 unless local_ver > remote_ver
         rescue BFS::FileNotFound
           nil
-        end if local_rev.positive?
+        end if local_ver.positive?
 
-        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+        stream.create metadata: { META_VERSION => local_ver.to_s } do |fmt|
           iter = enum.respond_to?(:find_each) ? :find_each : :each
           enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
         end

data/lib/feedx/stream.rb

@@ -8,8 +8,8 @@ module Feedx
 
     # Behaves like new, but accepts an optional block.
     # If a block is given, streams are automatically closed after the block is yielded.
-    def self.open(url, **opts)
-      stream = new(url, **opts)
+    def self.open(url, **)
+      stream = new(url, **)
       return stream unless block_given?
 
       begin

data/lib/feedx.rb

@@ -1,6 +1,6 @@
 module Feedx
-  META_LAST_MODIFIED = 'X-Feedx-Last-Modified'.freeze
-  META_LAST_MODIFIED_DC = META_LAST_MODIFIED.downcase.freeze
+  META_VERSION = 'X-Feedx-Version'.freeze
+  META_VERSION_DC = META_VERSION.downcase.freeze
 
   autoload :Cache, 'feedx/cache'
   autoload :Compression, 'feedx/compression'
@@ -9,5 +9,4 @@ module Feedx
   autoload :Stream, 'feedx/stream'
   autoload :Producer, 'feedx/producer'
   autoload :Pusher, 'feedx/pusher'
-  autoload :TaskState, 'feedx/task_state'
 end

data/manifest.go

@@ -0,0 +1,65 @@
+package feedx
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"strings"
+
+	"github.com/bsm/bfs"
+)
+
+// manifest holds the current feed status.
+// the current manifest is consumed before each push and a new manifest written after each push.
+type manifest struct {
+	// Version holds the most recent version of the records included in Files.
+	Version int64 `json:"version"`
+	// Generation is a incrementing counter for use in file compaction.
+	Generation int `json:"generation"`
+	// Files holds a set of data files
+	Files []string `json:"files"`
+}
+
+func loadManifest(ctx context.Context, obj *bfs.Object) (*manifest, error) {
+	m := new(manifest)
+
+	r, err := NewReader(ctx, obj, nil)
+	if errors.Is(err, bfs.ErrNotFound) {
+		return m, nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer func() { _ = r.Close() }()
+
+	if err := r.Decode(m); errors.Is(err, bfs.ErrNotFound) { // some BFS implementations defer Open-ing the S3 object till first Decode call
+		return m, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (m *manifest) newDataFileName(wopt *WriterOptions) string {
+	version := strings.ReplaceAll(strconv.FormatInt(wopt.Version, 10), ".", "")
+
+	formatExt := ".json"
+	switch wopt.Format {
+	case ProtobufFormat:
+		formatExt = ".pb"
+	case CBORFormat:
+		formatExt = ".cbor"
+	}
+
+	var compressionSuffix string
+	switch wopt.Compression {
+	case GZipCompression:
+		compressionSuffix = "z"
+	case FlateCompression:
+		compressionSuffix = ".flate"
+	case ZstdCompression:
+		compressionSuffix = ".zst"
+	}
+
+	return "data-" + strconv.Itoa(m.Generation) + "-" + version + formatExt + compressionSuffix
+}