feedx 0.12.0 → 0.12.6

data/internal/testdata/testdata.pb.go

@@ -1,24 +1,29 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.24.0-devel
+// 	protoc        v3.11.3
 // source: internal/testdata/testdata.proto
 
 package testdata
 
 import (
-	fmt "fmt"
-	proto "github.com/gogo/protobuf/proto"
-	math "math"
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
 )
 
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
 
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
 
 type MockEnum int32
 
@@ -27,98 +32,192 @@ const (
 	MockEnum_FIRST   MockEnum = 3
 )
 
-var MockEnum_name = map[int32]string{
-	0: "UNKNOWN",
-	3: "FIRST",
-}
+// Enum value maps for MockEnum.
+var (
+	MockEnum_name = map[int32]string{
+		0: "UNKNOWN",
+		3: "FIRST",
+	}
+	MockEnum_value = map[string]int32{
+		"UNKNOWN": 0,
+		"FIRST":   3,
+	}
+)
 
-var MockEnum_value = map[string]int32{
-	"UNKNOWN": 0,
-	"FIRST":   3,
+func (x MockEnum) Enum() *MockEnum {
+	p := new(MockEnum)
+	*p = x
+	return p
 }
 
 func (x MockEnum) String() string {
-	return proto.EnumName(MockEnum_name, int32(x))
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
 }
 
-func (MockEnum) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_076a9f61cb4a1904, []int{0}
+func (MockEnum) Descriptor() protoreflect.EnumDescriptor {
+	return file_internal_testdata_testdata_proto_enumTypes[0].Descriptor()
 }
 
-type MockMessage struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Enum                 MockEnum `protobuf:"varint,2,opt,name=enum,proto3,enum=feedx.internal.testdata.MockEnum" json:"enum,omitempty"`
-	Height               uint32   `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+func (MockEnum) Type() protoreflect.EnumType {
+	return &file_internal_testdata_testdata_proto_enumTypes[0]
 }
 
-func (m *MockMessage) Reset()         { *m = MockMessage{} }
-func (m *MockMessage) String() string { return proto.CompactTextString(m) }
-func (*MockMessage) ProtoMessage()    {}
-func (*MockMessage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_076a9f61cb4a1904, []int{0}
+func (x MockEnum) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
 }
-func (m *MockMessage) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_MockMessage.Unmarshal(m, b)
+
+// Deprecated: Use MockEnum.Descriptor instead.
+func (MockEnum) EnumDescriptor() ([]byte, []int) {
+	return file_internal_testdata_testdata_proto_rawDescGZIP(), []int{0}
 }
-func (m *MockMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_MockMessage.Marshal(b, m, deterministic)
+
+type MockMessage struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name   string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Enum   MockEnum `protobuf:"varint,2,opt,name=enum,proto3,enum=feedx.internal.testdata.MockEnum" json:"enum,omitempty"`
+	Height uint32   `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"`
 }
-func (m *MockMessage) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_MockMessage.Merge(m, src)
+
+func (x *MockMessage) Reset() {
+	*x = MockMessage{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_internal_testdata_testdata_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
 }
-func (m *MockMessage) XXX_Size() int {
-	return xxx_messageInfo_MockMessage.Size(m)
+
+func (x *MockMessage) String() string {
+	return protoimpl.X.MessageStringOf(x)
 }
-func (m *MockMessage) XXX_DiscardUnknown() {
-	xxx_messageInfo_MockMessage.DiscardUnknown(m)
+
+func (*MockMessage) ProtoMessage() {}
+
+func (x *MockMessage) ProtoReflect() protoreflect.Message {
+	mi := &file_internal_testdata_testdata_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
 }
 
-var xxx_messageInfo_MockMessage proto.InternalMessageInfo
+// Deprecated: Use MockMessage.ProtoReflect.Descriptor instead.
+func (*MockMessage) Descriptor() ([]byte, []int) {
+	return file_internal_testdata_testdata_proto_rawDescGZIP(), []int{0}
+}
 
-func (m *MockMessage) GetName() string {
-	if m != nil {
-		return m.Name
+func (x *MockMessage) GetName() string {
+	if x != nil {
+		return x.Name
 	}
 	return ""
 }
 
-func (m *MockMessage) GetEnum() MockEnum {
-	if m != nil {
-		return m.Enum
+func (x *MockMessage) GetEnum() MockEnum {
+	if x != nil {
+		return x.Enum
 	}
 	return MockEnum_UNKNOWN
 }
 
-func (m *MockMessage) GetHeight() uint32 {
-	if m != nil {
-		return m.Height
+func (x *MockMessage) GetHeight() uint32 {
+	if x != nil {
+		return x.Height
 	}
 	return 0
 }
 
-func init() {
-	proto.RegisterEnum("feedx.internal.testdata.MockEnum", MockEnum_name, MockEnum_value)
-	proto.RegisterType((*MockMessage)(nil), "feedx.internal.testdata.MockMessage")
-}
-
-func init() { proto.RegisterFile("internal/testdata/testdata.proto", fileDescriptor_076a9f61cb4a1904) }
-
-var fileDescriptor_076a9f61cb4a1904 = []byte{
-	// 199 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0xc8, 0xcc, 0x2b, 0x49,
-	0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x49, 0x2d, 0x2e, 0x49, 0x49, 0x2c, 0x49, 0x84, 0x33, 0xf4,
-	0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xc4, 0xd3, 0x52, 0x53, 0x53, 0x2a, 0xf4, 0x60, 0xea, 0xf4,
-	0x60, 0xd2, 0x4a, 0x05, 0x5c, 0xdc, 0xbe, 0xf9, 0xc9, 0xd9, 0xbe, 0xa9, 0xc5, 0xc5, 0x89, 0xe9,
-	0xa9, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41,
-	0x60, 0xb6, 0x90, 0x29, 0x17, 0x4b, 0x6a, 0x5e, 0x69, 0xae, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x9f,
-	0x91, 0xa2, 0x1e, 0x0e, 0xa3, 0xf4, 0x40, 0xe6, 0xb8, 0xe6, 0x95, 0xe6, 0x06, 0x81, 0x95, 0x0b,
-	0x89, 0x71, 0xb1, 0x65, 0xa4, 0x66, 0xa6, 0x67, 0x94, 0x48, 0x30, 0x2b, 0x30, 0x6a, 0xf0, 0x06,
-	0x41, 0x79, 0x5a, 0x4a, 0x5c, 0x1c, 0x30, 0x95, 0x42, 0xdc, 0x5c, 0xec, 0xa1, 0x7e, 0xde, 0x7e,
-	0xfe, 0xe1, 0x7e, 0x02, 0x0c, 0x42, 0x9c, 0x5c, 0xac, 0x6e, 0x9e, 0x41, 0xc1, 0x21, 0x02, 0xcc,
-	0x4e, 0x2a, 0x51, 0x4a, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0x60,
-	0x0b, 0xf5, 0x31, 0xfc, 0x98, 0xc4, 0x06, 0xf6, 0x9b, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x42,
-	0xf6, 0x49, 0xb3, 0xff, 0x00, 0x00, 0x00,
+var File_internal_testdata_testdata_proto protoreflect.FileDescriptor
+
+var file_internal_testdata_testdata_proto_rawDesc = []byte{
+	0x0a, 0x20, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64,
+	0x61, 0x74, 0x61, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x12, 0x17, 0x66, 0x65, 0x65, 0x64, 0x78, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x22, 0x70, 0x0a, 0x0b, 0x4d,
+	0x6f, 0x63, 0x6b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35,
+	0x0a, 0x04, 0x65, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66,
+	0x65, 0x65, 0x64, 0x78, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x75, 0x6d, 0x52,
+	0x04, 0x65, 0x6e, 0x75, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x2a, 0x22, 0x0a,
+	0x08, 0x4d, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b,
+	0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, 0x52, 0x53, 0x54, 0x10,
+	0x03, 0x42, 0x24, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x66, 0x65, 0x65, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x74,
+	0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_internal_testdata_testdata_proto_rawDescOnce sync.Once
+	file_internal_testdata_testdata_proto_rawDescData = file_internal_testdata_testdata_proto_rawDesc
+)
+
+func file_internal_testdata_testdata_proto_rawDescGZIP() []byte {
+	file_internal_testdata_testdata_proto_rawDescOnce.Do(func() {
+		file_internal_testdata_testdata_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_testdata_testdata_proto_rawDescData)
+	})
+	return file_internal_testdata_testdata_proto_rawDescData
+}
+
+var file_internal_testdata_testdata_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_internal_testdata_testdata_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_internal_testdata_testdata_proto_goTypes = []interface{}{
+	(MockEnum)(0),       // 0: feedx.internal.testdata.MockEnum
+	(*MockMessage)(nil), // 1: feedx.internal.testdata.MockMessage
+}
+var file_internal_testdata_testdata_proto_depIdxs = []int32{
+	0, // 0: feedx.internal.testdata.MockMessage.enum:type_name -> feedx.internal.testdata.MockEnum
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_internal_testdata_testdata_proto_init() }
+func file_internal_testdata_testdata_proto_init() {
+	if File_internal_testdata_testdata_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_internal_testdata_testdata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*MockMessage); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_internal_testdata_testdata_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_internal_testdata_testdata_proto_goTypes,
+		DependencyIndexes: file_internal_testdata_testdata_proto_depIdxs,
+		EnumInfos:         file_internal_testdata_testdata_proto_enumTypes,
+		MessageInfos:      file_internal_testdata_testdata_proto_msgTypes,
+	}.Build()
+	File_internal_testdata_testdata_proto = out.File
+	file_internal_testdata_testdata_proto_rawDesc = nil
+	file_internal_testdata_testdata_proto_goTypes = nil
+	file_internal_testdata_testdata_proto_depIdxs = nil
 }

data/lib/feedx/cache/memory.rb

@@ -3,6 +3,7 @@ require 'monitor'
 # Thread-safe in-memory cache. Use for testing only.
 class Feedx::Cache::Memory < Feedx::Cache::Abstract
   def initialize
+    super
     @monitor = Monitor.new
     @entries = {}
   end

data/lib/feedx/consumer.rb

@@ -19,10 +19,10 @@ module Feedx
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
     def initialize(url, klass, format_options: {}, cache: nil, **opts)
-      @klass  = klass
-      @stream = Feedx::Stream.new(url, **opts)
-      @cache  = cache
-      @opts   = opts.merge(format_options)
+      @klass = klass
+      @url = url
+      @opts = opts.merge(format_options)
+      @cache = cache
 
       return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
 
@@ -31,21 +31,24 @@ module Feedx
 
     # @return [Boolean] returns true if performed.
     def each(&block)
+      stream = Feedx::Stream.new(@url, **@opts)
       remote_rev = nil
 
       if @cache
-        metadata   = @stream.blob.info.metadata
+        metadata   = stream.blob.info.metadata
         local_rev  = @cache.read.to_i
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return false if remote_rev.positive? && remote_rev <= local_rev
       end
 
-      @stream.open do |fmt|
+      stream.open do |fmt|
         fmt.decode_each(@klass, **@opts, &block)
       end
       @cache.write(remote_rev) if @cache && remote_rev
 
       true
+    ensure
+      stream&.close
     end
   end
 end

data/lib/feedx/format.rb

@@ -27,7 +27,7 @@ module Feedx
           ext = File.extname(base)
           raise ArgumentError, 'unable to detect format' if ext.empty?
 
-          kind = _resolve(ext[1..-1]) || _resolve(ext[1..-2])
+          kind = _resolve(ext[1..]) || _resolve(ext[1..-2])
           return kind if kind
 
           base = base[0..-ext.size - 1]

data/lib/feedx/producer.rb

@@ -22,9 +22,9 @@ module Feedx
       @enum = enum || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
-      @stream   = Feedx::Stream.new(url, **opts)
+      @url  = url
+      @opts = opts.merge(format_options)
       @last_mod = last_modified
-      @opts     = opts.merge(format_options)
 
       return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
 
@@ -32,23 +32,25 @@ module Feedx
     end
 
     def perform
-      enum      = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod  = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
-
-      begin
-        metadata   = @stream.blob.info.metadata
-        remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
-        return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound
-        nil
-      end if local_rev.positive?
-
-      @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
-        iter = enum.respond_to?(:find_each) ? :find_each : :each
-        enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+      Feedx::Stream.open(@url, **@opts) do |stream|
+        enum = @enum.is_a?(Proc) ? @enum.call : @enum
+        last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+        local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
+
+        begin
+          metadata   = stream.blob.info.metadata
+          remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
+          return -1 unless local_rev > remote_rev
+        rescue BFS::FileNotFound
+          nil
+        end if local_rev.positive?
+
+        stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
+          iter = enum.respond_to?(:find_each) ? :find_each : :each
+          enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
+        end
+        stream.blob.info.size
       end
-      @stream.blob.info.size
     end
   end
 end

data/lib/feedx/stream.rb

@@ -6,6 +6,19 @@ module Feedx
   class Stream
     attr_reader :blob
 
+    # Behaves like new, but accepts an optional block.
+    # If a block is given, streams are automatically closed after the block is yielded.
+    def self.open(url, **opts)
+      stream = new(url, **opts)
+      return stream unless block_given?
+
+      begin
+        yield stream
+      ensure
+        stream.close
+      end
+    end
+
     # @param [String] url the blob URL.
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
@@ -15,18 +28,18 @@ module Feedx
       @format   = detect_format(format)
       @compress = detect_compress(compress)
       @opts     = opts
+
+      BFS.defer(self, :close)
     end
 
     # Opens the remote for reading.
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(**opts)
+    def open(**opts, &block)
       @blob.open(**opts) do |io|
         @compress.reader(io, **@opts) do |cio|
-          @format.decoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.decoder(cio, **@opts, &block)
         end
       end
     end
@@ -35,16 +48,19 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(**opts)
+    def create(**opts, &block)
       @blob.create(**opts) do |io|
         @compress.writer(io, **@opts) do |cio|
-          @format.encoder(cio, **@opts) do |fmt|
-            yield fmt
-          end
+          @format.encoder(cio, **@opts, &block)
         end
       end
     end
 
+    # Closes the underlying connection.
+    def close
+      @blob.close
+    end
+
     private
 
     def detect_format(val)