feedx 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c601b31c975bf931998f4c498b86f4e0308be42d0064b9d138d64dc58f2ddb4e
-  data.tar.gz: ab223f06c6b9f9bf6c5b000f1268ded7a86e10e36715d43628bb19d08ac7809f
+  metadata.gz: 7d1c12376b32188b646923935967905bb97ab8c86dde28236a0341ee878eb631
+  data.tar.gz: 3c76d4356a0f6aafcad0e8ea37a2159f8170f5d1cea4072753e3c9947afd7613
 SHA512:
-  metadata.gz: e5200ae2328cb6353f1262639aca392cc09f18e30f9023c8db9da5e81e7236cb00bb07b543b660497d7c98d5368dfe24f8dbc410ac5234ef82ce270d0cc253d9
-  data.tar.gz: 940fe6bf7295e7b24dc71cea752fa2850ed1fc0dac736559f8289f93704690393ca81e216d222385eb33e5674709d06c82ff108ee10e8dad8b6bf6f57e97ed51
+  metadata.gz: d55b8ee3620301c3ffc5c95d2895d00f54ee562250c4aa8236ed93fd99e552cd67c126feab4a17b063415518e46b113ef918491b19e37043b5d36fa9e09c2808
+  data.tar.gz: b8eee064c4e12e462097087431cdd9b435720eead2d7524c9cc7d44cb4a7dde410756f443b71cd5b24261d3b06c95f1d3975ddca6b6ff69692e0e9a77a742446

data/.travis.yml

@@ -1,4 +1,13 @@
-language: ruby
-rvm:
-  - 2.5
-  - 2.4
+matrix:
+  include:
+    - language: ruby
+      rvm:
+        - 2.5
+    - language: ruby
+      rvm:
+        - 2.4
+    - language: go
+      go:
+        - 1.11.x
+      env:
+        - GO111MODULE=on

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    feedx (0.3.1)
+    feedx (0.3.2)
       bfs (>= 0.3.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    bfs (0.3.4)
+    bfs (0.3.6)
     diff-lcs (1.3)
     google-protobuf (3.6.1)
     jaro_winkler (1.5.1)
@@ -19,7 +19,7 @@ GEM
       google-protobuf
     powerpack (0.1.2)
     rainbow (3.0.0)
-    rake (12.3.1)
+    rake (12.3.2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -33,7 +33,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubocop (0.60.0)
+    rubocop (0.61.1)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5, != 2.5.1.1)
@@ -56,4 +56,4 @@ DEPENDENCIES
   rubocop
 
 BUNDLED WITH
-   1.16.4
+   1.17.2

data/Makefile

@@ -0,0 +1,7 @@
+default: vet test
+
+test:
+	go test ./...
+
+vet:
+	go vet ./...

data/compression.go

@@ -0,0 +1,63 @@
+package feedx
+
+import (
+	"compress/gzip"
+	"io"
+	"path"
+)
+
+// Compression represents the data compression.
+type Compression interface {
+	// NewReader wraps a reader.
+	NewReader(io.Reader) (io.ReadCloser, error)
+	// NewWriter wraps a writer.
+	NewWriter(io.Writer) (io.WriteCloser, error)
+}
+
+// DetectCompression detects the compression type from a URL path or file name.
+func DetectCompression(name string) Compression {
+	if name != "" {
+		ext := path.Ext(path.Base(name))
+		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
+			return GZipCompression
+		}
+	}
+	return NoCompression
+}
+
+// --------------------------------------------------------------------
+
+// NoCompression is just a pass-through without compression.
+var NoCompression = noCompression{}
+
+type noCompression struct{}
+
+func (noCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return noCompressionWrapper{Reader: r}, nil
+}
+
+func (noCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return noCompressionWrapper{Writer: w}, nil
+}
+
+type noCompressionWrapper struct {
+	io.Reader
+	io.Writer
+}
+
+func (noCompressionWrapper) Close() error { return nil }
+
+// --------------------------------------------------------------------
+
+// GZipCompression supports gzip compression format.
+var GZipCompression = gzipCompression{}
+
+type gzipCompression struct{}
+
+func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return gzip.NewReader(r)
+}
+
+func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return gzip.NewWriter(w), nil
+}

data/compression_test.go

@@ -0,0 +1,66 @@
+package feedx_test
+
+import (
+	"bytes"
+
+	"github.com/bsm/feedx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Compression", func() {
+	var data = bytes.Repeat([]byte("wxyz"), 1024)
+
+	runSharedTest := func(subject feedx.Compression) {
+		buf := new(bytes.Buffer)
+
+		w, err := subject.NewWriter(buf)
+		Expect(err).NotTo(HaveOccurred())
+		defer w.Close()
+
+		Expect(w.Write(data)).To(Equal(4096))
+		Expect(w.Write(data)).To(Equal(4096))
+		Expect(w.Close()).To(Succeed())
+
+		r, err := subject.NewReader(buf)
+		Expect(err).NotTo(HaveOccurred())
+		defer r.Close()
+
+		p := make([]byte, 20)
+		Expect(r.Read(p)).To(Equal(20))
+		Expect(string(p)).To(Equal("wxyzwxyzwxyzwxyzwxyz"))
+		Expect(r.Close()).To(Succeed())
+	}
+
+	It("should detect the format", func() {
+		Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
+		Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
+		Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
+
+		Expect(feedx.DetectCompression("/path/to/file.pb")).To(Equal(feedx.NoCompression))
+		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
+		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
+
+		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
+		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
+		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
+	})
+
+	Describe("NoCompression", func() {
+		var subject = feedx.NoCompression
+		var _ feedx.Compression = subject
+
+		It("should write/read", func() {
+			runSharedTest(subject)
+		})
+	})
+
+	Describe("GZipCompression", func() {
+		var subject = feedx.GZipCompression
+		var _ feedx.Compression = subject
+
+		It("should write/read", func() {
+			runSharedTest(subject)
+		})
+	})
+})

data/consumer.go

@@ -0,0 +1,206 @@
+package feedx
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/bsm/bfs"
+)
+
+// ConsumerOptions configure the Puller instance.
+type ConsumerOptions struct {
+	// The interval used by Puller to check the remote changes.
+	// Default: 1m
+	Interval time.Duration
+
+	// Format specifies the format
+	// Default: auto-detected from URL path.
+	Format Format
+
+	// Compression specifies the compression type.
+	// Default: auto-detected from URL path.
+	Compression Compression
+
+	// AfterSync callbacks are triggered after each sync, receiving
+	// the updated status and error (if occurred).
+	AfterSync func(updated bool, err error)
+}
+
+func (o *ConsumerOptions) norm(name string) error {
+	if o.Interval <= 0 {
+		o.Interval = time.Minute
+	}
+	if o.Format == nil {
+		o.Format = DetectFormat(name)
+
+		if o.Format == nil {
+			return fmt.Errorf("feedx: unable to detect format from %q", name)
+		}
+	}
+	if o.Compression == nil {
+		o.Compression = DetectCompression(name)
+	}
+	return nil
+}
+
+// ParseFunc is a data parse function.
+type ParseFunc func(FormatDecoder) (data interface{}, size int64, err error)
+
+// Consumer manages data retrieval from a remote feed.
+// It queries the feed in regular intervals, continuously retrieving new updates.
+type Consumer interface {
+	// Data returns the data as returned by ParseFunc on last sync.
+	Data() interface{}
+	// LastCheck returns time of last sync attempt.
+	LastCheck() time.Time
+	// LastModified returns time at which the remote feed was last modified.
+	LastModified() time.Time
+	// Size returns the size as returned by ParseFunc on last sync.
+	Size() int64
+	// Close stops the underlying sync process.
+	Close() error
+}
+
+// NewConsumer starts a new feed consumer.
+func NewConsumer(ctx context.Context, srcURL string, opt *ConsumerOptions, parse ParseFunc) (Consumer, error) {
+	src, err := bfs.NewObject(ctx, srcURL)
+	if err != nil {
+		return nil, err
+	}
+
+	var o ConsumerOptions
+	if opt != nil {
+		o = *opt
+	}
+	if err := o.norm(src.Name()); err != nil {
+		_ = src.Close()
+		return nil, err
+	}
+
+	ctx, stop := context.WithCancel(ctx)
+	f := &consumer{
+		src:   src,
+		opt:   o,
+		ctx:   ctx,
+		stop:  stop,
+		parse: parse,
+	}
+
+	// run initial sync
+	if _, err := f.sync(true); err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+
+	// start continuous loop
+	go f.loop()
+
+	return f, nil
+}
+
+type consumer struct {
+	src  *bfs.Object
+	opt  ConsumerOptions
+	ctx  context.Context
+	stop context.CancelFunc
+
+	parse ParseFunc
+
+	size, lastModMs int64
+	data, lastCheck atomic.Value
+}
+
+// Data implements Feed interface.
+func (f *consumer) Data() interface{} {
+	return f.data.Load()
+}
+
+// Size implements Feed interface.
+func (f *consumer) Size() int64 {
+	return atomic.LoadInt64(&f.size)
+}
+
+// LastCheck implements Feed interface.
+func (f *consumer) LastCheck() time.Time {
+	return f.lastCheck.Load().(time.Time)
+}
+
+// LastModified implements Feed interface.
+func (f *consumer) LastModified() time.Time {
+	msec := atomic.LoadInt64(&f.lastModMs)
+	return time.Unix(msec/1000, msec%1000*1e6)
+}
+
+// Close implements Feed interface.
+func (f *consumer) Close() error {
+	f.stop()
+	return f.src.Close()
+}
+
+func (f *consumer) sync(force bool) (bool, error) {
+	f.lastCheck.Store(time.Now())
+
+	info, err := f.src.Head(f.ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// calculate last modified time
+	msec, _ := strconv.ParseInt(info.Metadata[lastModifiedMetaKey], 10, 64)
+
+	// skip update if not forced or modified
+	if msec == atomic.LoadInt64(&f.lastModMs) && !force {
+		return false, nil
+	}
+
+	// open remote for reading
+	r, err := f.src.Open(f.ctx)
+	if err != nil {
+		return false, err
+	}
+	defer r.Close()
+
+	// wrap in compressed reader
+	c, err := f.opt.Compression.NewReader(r)
+	if err != nil {
+		return false, err
+	}
+	defer c.Close()
+
+	// open decoder
+	d, err := f.opt.Format.NewDecoder(c)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	// parse feed
+	data, size, err := f.parse(d)
+	if err != nil {
+		return false, err
+	}
+
+	// update stores
+	f.data.Store(data)
+	atomic.StoreInt64(&f.size, size)
+	atomic.StoreInt64(&f.lastModMs, msec)
+	return true, nil
+}
+
+func (f *consumer) loop() {
+	ticker := time.NewTicker(f.opt.Interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-f.ctx.Done():
+			return
+		case <-ticker.C:
+			updated, err := f.sync(false)
+			f.opt.AfterSync(updated, err)
+		}
+	}
+}

data/consumer_test.go

@@ -0,0 +1,70 @@
+package feedx_test
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Consumer", func() {
+	ctx := context.Background()
+	msg := &tbp.Message{
+		Name:         "Joe",
+		TrueScotsman: true,
+		Hilarity:     tbp.Message_BILL_BAILEY,
+	}
+	pfn := func(dec feedx.FormatDecoder) (interface{}, int64, error) {
+		var msgs []*tbp.Message
+		for {
+			msg := new(tbp.Message)
+			if err := dec.Decode(msg); err == io.EOF {
+				break
+			} else if err != nil {
+				return nil, 0, err
+			}
+			msgs = append(msgs, msg)
+		}
+		return msgs, int64(len(msgs)), nil
+	}
+
+	BeforeEach(func() {
+		memStore = bfs.NewInMem()
+		w, err := memStore.Create(ctx, "path/to/file.jsonz", &bfs.WriteOptions{
+			Metadata: map[string]string{"x-feedx-pusher-last-modified": "1544477788899"},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		defer w.Close()
+
+		c, err := feedx.GZipCompression.NewWriter(w)
+		Expect(err).NotTo(HaveOccurred())
+		defer c.Close()
+
+		f, err := feedx.JSONFormat.NewEncoder(c)
+		Expect(err).NotTo(HaveOccurred())
+		defer f.Close()
+
+		Expect(f.Encode(msg)).To(Succeed())
+		Expect(f.Encode(msg)).To(Succeed())
+		Expect(f.Close()).To(Succeed())
+		Expect(c.Close()).To(Succeed())
+		Expect(w.Close()).To(Succeed())
+	})
+
+	It("should sync and retrieve feeds from remote", func() {
+		subject, err := feedx.NewConsumer(ctx, "mem:///path/to/file.jsonz", nil, pfn)
+		Expect(err).NotTo(HaveOccurred())
+		defer subject.Close()
+
+		Expect(subject.LastCheck()).To(BeTemporally("~", time.Now(), time.Second))
+		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1544477788, 0), time.Second))
+		Expect(subject.Size()).To(Equal(int64(2)))
+		Expect(subject.Data()).To(Equal([]*tbp.Message{msg, msg}))
+		Expect(subject.Close()).To(Succeed())
+	})
+})

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.3.1'
+  s.version       = '0.3.2'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/feedx.go

@@ -0,0 +1,3 @@
+package feedx
+
+const lastModifiedMetaKey = "x-feedx-pusher-last-modified"

data/feedx_test.go

@@ -0,0 +1,27 @@
+package feedx_test
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/bsm/bfs"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// ------------------------------------------------------------------------
+
+var memStore *bfs.InMem
+
+func init() {
+	memStore = bfs.NewInMem()
+	bfs.Register("mem", func(_ context.Context, u *url.URL) (bfs.Bucket, error) {
+		return memStore, nil
+	})
+}
+
+func TestSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "feedx")
+}

data/format.go

@@ -0,0 +1,122 @@
+package feedx
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"path"
+
+	"github.com/golang/protobuf/proto"
+
+	pbio "github.com/gogo/protobuf/io"
+)
+
+// Format represents the data format.
+type Format interface {
+	// NewDecoder wraps a decoder around a reader.
+	NewDecoder(io.Reader) (FormatDecoder, error)
+	// NewEncoder wraps an encoder around a writer.
+	NewEncoder(io.Writer) (FormatEncoder, error)
+}
+
+// DetectFormat detects the data format from a URL path or file name.
+// May return nil.
+func DetectFormat(name string) Format {
+	ext := path.Ext(path.Base(name))
+	switch ext {
+	case ".json":
+		return JSONFormat
+	case ".pb", ".proto", ".protobuf":
+		return ProtobufFormat
+	default:
+		if name != "" && ext != "" && ext[0] == '.' {
+			if ext[len(ext)-1] == 'z' {
+				return DetectFormat(name[0 : len(name)-1])
+			}
+			return DetectFormat(name[0 : len(name)-len(ext)])
+		}
+	}
+	return nil
+}
+
+// FormatDecoder methods
+type FormatDecoder interface {
+	// Decode decodes the next message into an interface.
+	Decode(v interface{}) error
+
+	io.Closer
+}
+
+// FormatEncoder methods
+type FormatEncoder interface {
+	// Encode encodes the value to the stream.
+	Encode(v interface{}) error
+
+	io.Closer
+}
+
+// --------------------------------------------------------------------
+
+// JSONFormat provides a Format implemention for JSON.
+var JSONFormat = jsonFormat{}
+
+type jsonFormat struct{}
+
+// NewDecoder implements Format.
+func (jsonFormat) NewDecoder(r io.Reader) (FormatDecoder, error) {
+	return jsonDecoderWrapper{Decoder: json.NewDecoder(r)}, nil
+}
+
+// NewEncoder implements Format.
+func (jsonFormat) NewEncoder(w io.Writer) (FormatEncoder, error) {
+	return jsonEncoderWrapper{Encoder: json.NewEncoder(w)}, nil
+}
+
+type jsonDecoderWrapper struct{ *json.Decoder }
+
+func (jsonDecoderWrapper) Close() error { return nil }
+
+type jsonEncoderWrapper struct{ *json.Encoder }
+
+func (jsonEncoderWrapper) Close() error { return nil }
+
+// --------------------------------------------------------------------
+
+const protobufMaxMessageSize = 20 * 1024 * 1024 // 20MB
+
+// ProtobufFormat provides a Format implemention for Protobuf.
+var ProtobufFormat = protobufFormat{}
+
+type protobufFormat struct{}
+
+// NewDecoder implements Format.
+func (protobufFormat) NewDecoder(r io.Reader) (FormatDecoder, error) {
+	rc := pbio.NewDelimitedReader(r, protobufMaxMessageSize)
+	return protobufDecoderWrapper{ReadCloser: rc}, nil
+}
+
+// NewEncoder implements Format.
+func (protobufFormat) NewEncoder(w io.Writer) (FormatEncoder, error) {
+	wc := pbio.NewDelimitedWriter(w)
+	return protobufEncoderWrapper{WriteCloser: wc}, nil
+}
+
+type protobufDecoderWrapper struct{ pbio.ReadCloser }
+
+func (w protobufDecoderWrapper) Decode(v interface{}) error {
+	msg, ok := v.(proto.Message)
+	if !ok {
+		return fmt.Errorf("feedx: value %v is not a proto.Message", v)
+	}
+	return w.ReadCloser.ReadMsg(msg)
+}
+
+type protobufEncoderWrapper struct{ pbio.WriteCloser }
+
+func (w protobufEncoderWrapper) Encode(v interface{}) error {
+	msg, ok := v.(proto.Message)
+	if !ok {
+		return fmt.Errorf("feedx: value %v is not a proto.Message", v)
+	}
+	return w.WriteMsg(msg)
+}

data/format_test.go

@@ -0,0 +1,80 @@
+package feedx_test
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/bsm/feedx"
+	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Format", func() {
+	msg := &tbp.Message{
+		Name:         "Joe",
+		TrueScotsman: true,
+		Hilarity:     tbp.Message_BILL_BAILEY,
+	}
+
+	runSharedTest := func(subject feedx.Format) {
+		buf := new(bytes.Buffer)
+
+		enc, err := subject.NewEncoder(buf)
+		Expect(err).NotTo(HaveOccurred())
+		defer enc.Close()
+
+		Expect(enc.Encode(msg)).To(Succeed())
+		Expect(enc.Encode(msg)).To(Succeed())
+		Expect(enc.Close()).To(Succeed())
+
+		dec, err := subject.NewDecoder(buf)
+		Expect(err).NotTo(HaveOccurred())
+		defer dec.Close()
+
+		v1 := new(tbp.Message)
+		Expect(dec.Decode(v1)).To(Succeed())
+		Expect(v1.Name).To(Equal("Joe"))
+
+		v2 := new(tbp.Message)
+		Expect(dec.Decode(v2)).To(Succeed())
+		Expect(v2.Name).To(Equal("Joe"))
+
+		v3 := new(tbp.Message)
+		Expect(dec.Decode(v3)).To(MatchError(io.EOF))
+
+		Expect(dec.Close()).To(Succeed())
+	}
+
+	It("should detect the format", func() {
+		Expect(feedx.DetectFormat("/path/to/file.json")).To(Equal(feedx.JSONFormat))
+		Expect(feedx.DetectFormat("/path/to/file.json.gz")).To(Equal(feedx.JSONFormat))
+		Expect(feedx.DetectFormat("/path/to/file.jsonz")).To(Equal(feedx.JSONFormat))
+
+		Expect(feedx.DetectFormat("/path/to/file.pb")).To(Equal(feedx.ProtobufFormat))
+		Expect(feedx.DetectFormat("/path/to/file.pb.gz")).To(Equal(feedx.ProtobufFormat))
+		Expect(feedx.DetectFormat("/path/to/file.pbz")).To(Equal(feedx.ProtobufFormat))
+
+		Expect(feedx.DetectFormat("")).To(BeNil())
+		Expect(feedx.DetectFormat("/path/to/file")).To(BeNil())
+		Expect(feedx.DetectFormat("/path/to/file.txt")).To(BeNil())
+	})
+
+	Describe("JSONFormat", func() {
+		var subject = feedx.JSONFormat
+		var _ feedx.Format = subject
+
+		It("should encode/decode", func() {
+			runSharedTest(subject)
+		})
+	})
+
+	Describe("ProtobufFormat", func() {
+		var subject = feedx.ProtobufFormat
+		var _ feedx.Format = subject
+
+		It("should encode/decode", func() {
+			runSharedTest(subject)
+		})
+	})
+})

data/go.mod

@@ -0,0 +1,13 @@
+module github.com/bsm/feedx
+
+require (
+	github.com/bsm/bfs v0.4.0
+	github.com/gogo/protobuf v1.2.0
+	github.com/golang/protobuf v1.2.0
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/gomega v1.4.3
+	golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6 // indirect
+	golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect
+	gopkg.in/yaml.v2 v2.2.2 // indirect
+)

data/go.sum

@@ -0,0 +1,111 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.32.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+git.apache.org/thrift.git v0.0.0-20181106172052-f7d43ce0aa58/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/aws/aws-sdk-go v1.15.71/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
+github.com/bsm/bfs v0.4.0 h1:+fT6iDNt1ytYVbSA1PCJFQez8j+DuO53D38wy39ol1g=
+github.com/bsm/bfs v0.4.0/go.mod h1:b96pNJ2nox+3JhdQL2oaVrzSsTpd1pEL+GgQuWjxIO0=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849 h1:FSqE2GGG7wzsYUsWiQ8MZrvEd1EOyU3NCF0AW3Wtltg=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6 h1:gT0Y6H7hbVPUtvtk0YGxMXPgN+p8fYlqWkgJeUCZcaQ=
+golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8 h1:YoY1wS6JYVRpIfFngRf2HHo9R9dAne3xbkGOQ5rJXjU=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM=
+golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181107225058-a28dfb48e06b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181108001712-cfbc873f6b93/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.39.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-29 00:00:00.000000000 Z
+date: 2018-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -108,9 +108,20 @@ files:
 - Gemfile
 - Gemfile.lock
 - LICENSE
+- Makefile
 - README.md
 - Rakefile
+- compression.go
+- compression_test.go
+- consumer.go
+- consumer_test.go
 - feedx.gemspec
+- feedx.go
+- feedx_test.go
+- format.go
+- format_test.go
+- go.mod
+- go.sum
 - lib/feedx.rb
 - lib/feedx/compression.rb
 - lib/feedx/compression/abstract.rb
@@ -149,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds