feedx 0.12.5 → 0.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +10 -29
- data/compression_test.go +4 -4
- data/consumer.go +12 -3
- data/consumer_test.go +48 -17
- data/ext/parquet/decoder_test.go +2 -2
- data/ext/parquet/go.mod +1 -1
- data/ext/parquet/go.sum +2 -0
- data/feedx.gemspec +1 -1
- data/feedx_ext_test.go +6 -0
- data/feedx_test.go +4 -4
- data/format_test.go +3 -3
- data/producer_test.go +2 -2
- data/reader_test.go +4 -3
- data/writer.go +1 -4
- data/writer_test.go +6 -6
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab1b8d12efde331dcd1ca96c686c29e1ef16ab895f049121078b9f0503d23276
|
4
|
+
data.tar.gz: 3d51319d3ea39cbceeb480252e06c37238fce0cda68aa6b65cf1f59c0f0af38b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d08c34b59158c69d500b7cffe8019550dafa87b760e4b0cb3b25afc367801a3f3473486987dcb62e5c1bc62c05d3653bf26692fcbdcbb552d2af1073f4eced50
|
7
|
+
data.tar.gz: cfa3f24458cead0c12c1f03b559ded816095e1e9cf29a1918c142cd255f782fa3c51d7c45ca5833ab25b5a745e0e68f4b1b1062c7760c56ff1b57da122767135
|
data/Gemfile.lock
CHANGED
@@ -1,22 +1,15 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
feedx (0.12.
|
4
|
+
feedx (0.12.6)
|
5
5
|
bfs (>= 0.8.0)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
|
-
activesupport (6.1.3.1)
|
11
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
12
|
-
i18n (>= 1.6, < 2)
|
13
|
-
minitest (>= 5.1)
|
14
|
-
tzinfo (~> 2.0)
|
15
|
-
zeitwerk (~> 2.3)
|
16
10
|
ast (2.4.2)
|
17
|
-
bfs (0.8.
|
11
|
+
bfs (0.8.4)
|
18
12
|
bigdecimal (3.0.2)
|
19
|
-
concurrent-ruby (1.1.8)
|
20
13
|
diff-lcs (1.4.4)
|
21
14
|
extpp (0.0.9)
|
22
15
|
gio2 (3.4.4)
|
@@ -26,18 +19,14 @@ GEM
|
|
26
19
|
pkg-config (>= 1.3.5)
|
27
20
|
gobject-introspection (3.4.4)
|
28
21
|
glib2 (= 3.4.4)
|
29
|
-
google-protobuf (3.
|
30
|
-
i18n (1.8.10)
|
31
|
-
concurrent-ruby (~> 1.0)
|
32
|
-
minitest (5.14.4)
|
22
|
+
google-protobuf (3.17.3)
|
33
23
|
native-package-installer (1.1.1)
|
34
24
|
parallel (1.20.1)
|
35
25
|
parser (3.0.1.1)
|
36
26
|
ast (~> 2.4.1)
|
37
|
-
pbio (0.
|
27
|
+
pbio (0.3.0)
|
38
28
|
google-protobuf
|
39
29
|
pkg-config (1.4.6)
|
40
|
-
rack (2.2.3)
|
41
30
|
rainbow (3.0.0)
|
42
31
|
rake (13.0.3)
|
43
32
|
red-arrow (3.0.0)
|
@@ -63,40 +52,32 @@ GEM
|
|
63
52
|
diff-lcs (>= 1.2.0, < 2.0)
|
64
53
|
rspec-support (~> 3.10.0)
|
65
54
|
rspec-support (3.10.2)
|
66
|
-
rubocop (1.
|
55
|
+
rubocop (1.17.0)
|
67
56
|
parallel (~> 1.10)
|
68
57
|
parser (>= 3.0.0.0)
|
69
58
|
rainbow (>= 2.2.2, < 4.0)
|
70
59
|
regexp_parser (>= 1.8, < 3.0)
|
71
60
|
rexml
|
72
|
-
rubocop-ast (>= 1.
|
61
|
+
rubocop-ast (>= 1.7.0, < 2.0)
|
73
62
|
ruby-progressbar (~> 1.7)
|
74
63
|
unicode-display_width (>= 1.4.0, < 3.0)
|
75
|
-
rubocop-ast (1.
|
64
|
+
rubocop-ast (1.7.0)
|
76
65
|
parser (>= 3.0.1.1)
|
77
|
-
rubocop-bsm (0.
|
66
|
+
rubocop-bsm (0.6.0)
|
78
67
|
rubocop (~> 1.0)
|
79
68
|
rubocop-performance
|
80
|
-
rubocop-rails
|
81
69
|
rubocop-rake
|
82
70
|
rubocop-rspec
|
83
|
-
rubocop-performance (1.11.
|
71
|
+
rubocop-performance (1.11.3)
|
84
72
|
rubocop (>= 1.7.0, < 2.0)
|
85
73
|
rubocop-ast (>= 0.4.0)
|
86
|
-
rubocop-rails (2.9.1)
|
87
|
-
activesupport (>= 4.2.0)
|
88
|
-
rack (>= 1.1)
|
89
|
-
rubocop (>= 0.90.0, < 2.0)
|
90
74
|
rubocop-rake (0.5.1)
|
91
75
|
rubocop
|
92
|
-
rubocop-rspec (2.
|
76
|
+
rubocop-rspec (2.4.0)
|
93
77
|
rubocop (~> 1.0)
|
94
78
|
rubocop-ast (>= 1.1.0)
|
95
79
|
ruby-progressbar (1.11.0)
|
96
|
-
tzinfo (2.0.4)
|
97
|
-
concurrent-ruby (~> 1.0)
|
98
80
|
unicode-display_width (2.0.0)
|
99
|
-
zeitwerk (2.4.2)
|
100
81
|
|
101
82
|
PLATFORMS
|
102
83
|
ruby
|
data/compression_test.go
CHANGED
@@ -32,7 +32,7 @@ var _ = Describe("Compression", func() {
|
|
32
32
|
Expect(r.Close()).To(Succeed())
|
33
33
|
}
|
34
34
|
|
35
|
-
It("
|
35
|
+
It("detects the format", func() {
|
36
36
|
Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
|
37
37
|
Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
|
38
38
|
Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
|
@@ -53,7 +53,7 @@ var _ = Describe("Compression", func() {
|
|
53
53
|
var subject = feedx.NoCompression
|
54
54
|
var _ feedx.Compression = subject
|
55
55
|
|
56
|
-
It("
|
56
|
+
It("writes/reads", func() {
|
57
57
|
runSharedTest(subject)
|
58
58
|
})
|
59
59
|
})
|
@@ -62,7 +62,7 @@ var _ = Describe("Compression", func() {
|
|
62
62
|
var subject = feedx.GZipCompression
|
63
63
|
var _ feedx.Compression = subject
|
64
64
|
|
65
|
-
It("
|
65
|
+
It("writes/reads", func() {
|
66
66
|
runSharedTest(subject)
|
67
67
|
})
|
68
68
|
})
|
@@ -71,7 +71,7 @@ var _ = Describe("Compression", func() {
|
|
71
71
|
var subject = feedx.FlateCompression
|
72
72
|
var _ feedx.Compression = subject
|
73
73
|
|
74
|
-
It("
|
74
|
+
It("writes/reads", func() {
|
75
75
|
runSharedTest(subject)
|
76
76
|
})
|
77
77
|
})
|
data/consumer.go
CHANGED
@@ -50,6 +50,8 @@ type Consumer interface {
|
|
50
50
|
Data() interface{}
|
51
51
|
// LastSync returns time of last sync attempt.
|
52
52
|
LastSync() time.Time
|
53
|
+
// LastConsumed returns time of last feed consumption.
|
54
|
+
LastConsumed() time.Time
|
53
55
|
// LastModified returns time at which the remote feed was last modified.
|
54
56
|
LastModified() time.Time
|
55
57
|
// NumRead returns the number of values consumed during the last sync.
|
@@ -114,7 +116,7 @@ type consumer struct {
|
|
114
116
|
cfn ConsumeFunc
|
115
117
|
data atomic.Value
|
116
118
|
|
117
|
-
numRead, lastMod, lastSync int64
|
119
|
+
numRead, lastMod, lastSync, lastConsumed int64
|
118
120
|
}
|
119
121
|
|
120
122
|
// Data implements Consumer interface.
|
@@ -132,6 +134,11 @@ func (c *consumer) LastSync() time.Time {
|
|
132
134
|
return timestamp(atomic.LoadInt64(&c.lastSync)).Time()
|
133
135
|
}
|
134
136
|
|
137
|
+
// LastConsumed implements Consumer interface.
|
138
|
+
func (c *consumer) LastConsumed() time.Time {
|
139
|
+
return timestamp(atomic.LoadInt64(&c.lastConsumed)).Time()
|
140
|
+
}
|
141
|
+
|
135
142
|
// LastModified implements Consumer interface.
|
136
143
|
func (c *consumer) LastModified() time.Time {
|
137
144
|
return timestamp(atomic.LoadInt64(&c.lastMod)).Time()
|
@@ -147,8 +154,9 @@ func (c *consumer) Close() error {
|
|
147
154
|
}
|
148
155
|
|
149
156
|
func (c *consumer) sync(force bool) (*ConsumerSync, error) {
|
157
|
+
syncTime := timestampFromTime(time.Now()).Millis()
|
150
158
|
defer func() {
|
151
|
-
atomic.StoreInt64(&c.lastSync,
|
159
|
+
atomic.StoreInt64(&c.lastSync, syncTime)
|
152
160
|
}()
|
153
161
|
|
154
162
|
// retrieve original last modified time
|
@@ -158,7 +166,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
|
|
158
166
|
}
|
159
167
|
|
160
168
|
// skip update if not forced or modified
|
161
|
-
if lastMod.Millis() == atomic.LoadInt64(&c.lastMod)
|
169
|
+
if !force && lastMod > 0 && lastMod.Millis() == atomic.LoadInt64(&c.lastMod) {
|
162
170
|
return &ConsumerSync{Consumer: c}, nil
|
163
171
|
}
|
164
172
|
|
@@ -180,6 +188,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
|
|
180
188
|
c.data.Store(data)
|
181
189
|
atomic.StoreInt64(&c.numRead, int64(reader.NumRead()))
|
182
190
|
atomic.StoreInt64(&c.lastMod, lastMod.Millis())
|
191
|
+
atomic.StoreInt64(&c.lastConsumed, syncTime)
|
183
192
|
return &ConsumerSync{
|
184
193
|
Consumer: c,
|
185
194
|
Updated: true,
|
data/consumer_test.go
CHANGED
@@ -17,25 +17,26 @@ var _ = Describe("Consumer", func() {
|
|
17
17
|
var obj *bfs.Object
|
18
18
|
var ctx = context.Background()
|
19
19
|
|
20
|
+
consume := func(r *feedx.Reader) (interface{}, error) {
|
21
|
+
var msgs []*testdata.MockMessage
|
22
|
+
for {
|
23
|
+
var msg testdata.MockMessage
|
24
|
+
if err := r.Decode(&msg); err == io.EOF {
|
25
|
+
break
|
26
|
+
} else if err != nil {
|
27
|
+
return nil, err
|
28
|
+
}
|
29
|
+
msgs = append(msgs, &msg)
|
30
|
+
}
|
31
|
+
return msgs, nil
|
32
|
+
}
|
33
|
+
|
20
34
|
BeforeEach(func() {
|
21
35
|
obj = bfs.NewInMemObject("path/to/file.jsonz")
|
22
|
-
Expect(writeMulti(obj, 2)).To(Succeed())
|
36
|
+
Expect(writeMulti(obj, 2, mockTime)).To(Succeed())
|
23
37
|
|
24
38
|
var err error
|
25
|
-
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil,
|
26
|
-
var msgs []*testdata.MockMessage
|
27
|
-
for {
|
28
|
-
var msg testdata.MockMessage
|
29
|
-
if err := r.Decode(&msg); err == io.EOF {
|
30
|
-
break
|
31
|
-
}
|
32
|
-
if err != nil {
|
33
|
-
return nil, err
|
34
|
-
}
|
35
|
-
msgs = append(msgs, &msg)
|
36
|
-
}
|
37
|
-
return msgs, nil
|
38
|
-
})
|
39
|
+
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, consume)
|
39
40
|
Expect(err).NotTo(HaveOccurred())
|
40
41
|
})
|
41
42
|
|
@@ -43,11 +44,41 @@ var _ = Describe("Consumer", func() {
|
|
43
44
|
Expect(subject.Close()).To(Succeed())
|
44
45
|
})
|
45
46
|
|
46
|
-
It("
|
47
|
+
It("syncs/retrieves feeds from remote", func() {
|
47
48
|
Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
|
48
|
-
Expect(subject.
|
49
|
+
Expect(subject.LastConsumed()).To(BeTemporally("==", subject.LastSync()))
|
50
|
+
Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
|
49
51
|
Expect(subject.NumRead()).To(Equal(2))
|
50
52
|
Expect(subject.Data()).To(ConsistOf(seed(), seed()))
|
51
53
|
Expect(subject.Close()).To(Succeed())
|
52
54
|
})
|
55
|
+
|
56
|
+
It("consumes feeds only if necessary", func() {
|
57
|
+
prevSync := subject.LastSync()
|
58
|
+
time.Sleep(2 * time.Millisecond)
|
59
|
+
|
60
|
+
testable := subject.(interface{ TestSync() error })
|
61
|
+
Expect(testable.TestSync()).To(Succeed())
|
62
|
+
Expect(subject.LastSync()).To(BeTemporally(">", prevSync))
|
63
|
+
Expect(subject.LastConsumed()).To(BeTemporally("==", prevSync)) // skipped on last sync
|
64
|
+
Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
|
65
|
+
Expect(subject.NumRead()).To(Equal(2))
|
66
|
+
})
|
67
|
+
|
68
|
+
It("always consumes if LastModified not set", func() {
|
69
|
+
noModTime := bfs.NewInMemObject("path/to/file.json")
|
70
|
+
Expect(writeMulti(noModTime, 2, time.Time{})).To(Succeed())
|
71
|
+
|
72
|
+
csmr, err := feedx.NewConsumerForRemote(ctx, noModTime, nil, consume)
|
73
|
+
Expect(err).NotTo(HaveOccurred())
|
74
|
+
|
75
|
+
prevSync := csmr.LastSync()
|
76
|
+
time.Sleep(2 * time.Millisecond)
|
77
|
+
|
78
|
+
testable := csmr.(interface{ TestSync() error })
|
79
|
+
Expect(testable.TestSync()).To(Succeed())
|
80
|
+
Expect(csmr.LastSync()).To(BeTemporally(">", prevSync))
|
81
|
+
Expect(csmr.LastConsumed()).To(BeTemporally("==", csmr.LastSync())) // consumed on last sync
|
82
|
+
Expect(csmr.LastModified()).To(BeTemporally("==", time.Unix(0, 0)))
|
83
|
+
})
|
53
84
|
})
|
data/ext/parquet/decoder_test.go
CHANGED
@@ -34,7 +34,7 @@ var _ = Describe("Decoder", func() {
|
|
34
34
|
Expect(fixture.Close()).To(Succeed())
|
35
35
|
})
|
36
36
|
|
37
|
-
It("
|
37
|
+
It("decodes", func() {
|
38
38
|
v1 := new(mockStruct)
|
39
39
|
Expect(subject.Decode(v1)).To(Succeed())
|
40
40
|
Expect(v1).To(Equal(&mockStruct{
|
@@ -77,7 +77,7 @@ var _ = Describe("Decoder", func() {
|
|
77
77
|
Expect(subject.Decode(v9)).To(MatchError(io.EOF))
|
78
78
|
})
|
79
79
|
|
80
|
-
It("
|
80
|
+
It("opens from non-file readers", func() {
|
81
81
|
bin, err := ioutil.ReadFile("testdata/alltypes_plain.parquet")
|
82
82
|
Expect(err).NotTo(HaveOccurred())
|
83
83
|
|
data/ext/parquet/go.mod
CHANGED
@@ -3,7 +3,7 @@ module github.com/bsm/feedx/ext/parquet
|
|
3
3
|
go 1.15
|
4
4
|
|
5
5
|
require (
|
6
|
-
github.com/bsm/feedx v0.
|
6
|
+
github.com/bsm/feedx v0.12.5
|
7
7
|
github.com/bsm/ginkgo v1.16.1
|
8
8
|
github.com/bsm/gomega v1.11.0
|
9
9
|
github.com/bsm/parquet-go v0.0.0-20200904113737-792434824f00
|
data/ext/parquet/go.sum
CHANGED
@@ -4,6 +4,8 @@ github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkN
|
|
4
4
|
github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
|
5
5
|
github.com/bsm/bfs v0.11.3 h1:BTFCftgmuVZwwu6vyjhyKr/Pg1E+cZ5tLodj3wKxr94=
|
6
6
|
github.com/bsm/bfs v0.11.3/go.mod h1:sUhBrbc9g0XThRRrT9hiinMhhKbkKIdhLkFljk4fuzM=
|
7
|
+
github.com/bsm/feedx v0.12.5 h1:N751MFTRKRgoP5eG1S30EoUuHsI7aAxJ0aQZg+7xf/Y=
|
8
|
+
github.com/bsm/feedx v0.12.5/go.mod h1:l5YNhFomuWy9du+8+hznXMH8Hug2qSMik7b5Vipcy58=
|
7
9
|
github.com/bsm/ginkgo v1.16.0/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
|
8
10
|
github.com/bsm/ginkgo v1.16.1 h1:jp1v1dbmbGZDWmnGXDTN+XK3U1fTTNja9xYa7VBI0l0=
|
9
11
|
github.com/bsm/ginkgo v1.16.1/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
|
data/feedx.gemspec
CHANGED
data/feedx_ext_test.go
ADDED
data/feedx_test.go
CHANGED
@@ -32,12 +32,12 @@ func seed() *testdata.MockMessage {
|
|
32
32
|
}
|
33
33
|
}
|
34
34
|
|
35
|
+
var mockTime = time.Unix(1515151515, 123456789)
|
36
|
+
|
35
37
|
// ------------------------------------------------------------------------
|
36
38
|
|
37
|
-
func writeMulti(obj *bfs.Object, numEntries int) error {
|
38
|
-
w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
|
39
|
-
LastMod: time.Unix(1515151515, 123456789),
|
40
|
-
})
|
39
|
+
func writeMulti(obj *bfs.Object, numEntries int, lastMod time.Time) error {
|
40
|
+
w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{LastMod: lastMod})
|
41
41
|
defer w.Discard()
|
42
42
|
|
43
43
|
for i := 0; i < numEntries; i++ {
|
data/format_test.go
CHANGED
@@ -40,7 +40,7 @@ var _ = Describe("Format", func() {
|
|
40
40
|
Expect(dec.Close()).To(Succeed())
|
41
41
|
}
|
42
42
|
|
43
|
-
It("
|
43
|
+
It("detects the format", func() {
|
44
44
|
Expect(feedx.DetectFormat("/path/to/file.json")).To(Equal(feedx.JSONFormat))
|
45
45
|
Expect(feedx.DetectFormat("/path/to/file.json.gz")).To(Equal(feedx.JSONFormat))
|
46
46
|
Expect(feedx.DetectFormat("/path/to/file.json.flate")).To(Equal(feedx.JSONFormat))
|
@@ -60,7 +60,7 @@ var _ = Describe("Format", func() {
|
|
60
60
|
var subject = feedx.JSONFormat
|
61
61
|
var _ feedx.Format = subject
|
62
62
|
|
63
|
-
It("
|
63
|
+
It("encodes/decodes", func() {
|
64
64
|
runSharedTest(subject)
|
65
65
|
})
|
66
66
|
})
|
@@ -69,7 +69,7 @@ var _ = Describe("Format", func() {
|
|
69
69
|
var subject = feedx.ProtobufFormat
|
70
70
|
var _ feedx.Format = subject
|
71
71
|
|
72
|
-
It("
|
72
|
+
It("encodes/decodes", func() {
|
73
73
|
runSharedTest(subject)
|
74
74
|
})
|
75
75
|
})
|
data/producer_test.go
CHANGED
@@ -43,7 +43,7 @@ var _ = Describe("Producer", func() {
|
|
43
43
|
}
|
44
44
|
})
|
45
45
|
|
46
|
-
It("
|
46
|
+
It("produces", func() {
|
47
47
|
setup(nil)
|
48
48
|
|
49
49
|
Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
|
@@ -56,7 +56,7 @@ var _ = Describe("Producer", func() {
|
|
56
56
|
Expect(info.Size).To(BeNumerically("~", 75, 10))
|
57
57
|
})
|
58
58
|
|
59
|
-
It("
|
59
|
+
It("produces with custom last-mod check", func() {
|
60
60
|
setup(&feedx.ProducerOptions{
|
61
61
|
Interval: 50 * time.Millisecond,
|
62
62
|
LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
|
data/reader_test.go
CHANGED
@@ -4,6 +4,7 @@ import (
|
|
4
4
|
"context"
|
5
5
|
"io"
|
6
6
|
"io/ioutil"
|
7
|
+
"time"
|
7
8
|
|
8
9
|
"github.com/bsm/bfs"
|
9
10
|
"github.com/bsm/feedx"
|
@@ -19,7 +20,7 @@ var _ = Describe("Reader", func() {
|
|
19
20
|
|
20
21
|
BeforeEach(func() {
|
21
22
|
obj = bfs.NewInMemObject("path/to/file.json")
|
22
|
-
Expect(writeMulti(obj, 3)).To(Succeed())
|
23
|
+
Expect(writeMulti(obj, 3, time.Time{})).To(Succeed())
|
23
24
|
|
24
25
|
var err error
|
25
26
|
subject, err = feedx.NewReader(ctx, obj, nil)
|
@@ -30,14 +31,14 @@ var _ = Describe("Reader", func() {
|
|
30
31
|
Expect(subject.Close()).To(Succeed())
|
31
32
|
})
|
32
33
|
|
33
|
-
It("
|
34
|
+
It("reads", func() {
|
34
35
|
data, err := ioutil.ReadAll(subject)
|
35
36
|
Expect(err).NotTo(HaveOccurred())
|
36
37
|
Expect(len(data)).To(BeNumerically("~", 110, 20))
|
37
38
|
Expect(subject.NumRead()).To(Equal(0))
|
38
39
|
})
|
39
40
|
|
40
|
-
It("
|
41
|
+
It("decodes", func() {
|
41
42
|
var msgs []*testdata.MockMessage
|
42
43
|
for {
|
43
44
|
var msg testdata.MockMessage
|
data/writer.go
CHANGED
@@ -20,7 +20,7 @@ type WriterOptions struct {
|
|
20
20
|
Compression Compression
|
21
21
|
|
22
22
|
// Provides an optional last modified timestamp which is stored with the remote metadata.
|
23
|
-
// Default: time.
|
23
|
+
// Default: time.Time{}.
|
24
24
|
LastMod time.Time
|
25
25
|
}
|
26
26
|
|
@@ -31,9 +31,6 @@ func (o *WriterOptions) norm(name string) {
|
|
31
31
|
if o.Compression == nil {
|
32
32
|
o.Compression = DetectCompression(name)
|
33
33
|
}
|
34
|
-
if o.LastMod.IsZero() {
|
35
|
-
o.LastMod = time.Now()
|
36
|
-
}
|
37
34
|
}
|
38
35
|
|
39
36
|
// Writer encodes feeds to remote locations.
|
data/writer_test.go
CHANGED
@@ -20,7 +20,7 @@ var _ = Describe("Writer", func() {
|
|
20
20
|
compressed = bfs.NewInMemObject("path/to/file.jsonz")
|
21
21
|
})
|
22
22
|
|
23
|
-
It("
|
23
|
+
It("writes plain", func() {
|
24
24
|
w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
|
25
25
|
LastMod: time.Unix(1515151515, 123456789),
|
26
26
|
})
|
@@ -35,7 +35,7 @@ var _ = Describe("Writer", func() {
|
|
35
35
|
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
36
36
|
})
|
37
37
|
|
38
|
-
It("
|
38
|
+
It("writes compressed", func() {
|
39
39
|
w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
|
40
40
|
LastMod: time.Unix(1515151515, 123456789),
|
41
41
|
})
|
@@ -50,14 +50,14 @@ var _ = Describe("Writer", func() {
|
|
50
50
|
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
51
51
|
})
|
52
52
|
|
53
|
-
It("
|
54
|
-
Expect(writeMulti(plain, 10)).To(Succeed())
|
55
|
-
Expect(writeMulti(compressed, 10)).To(Succeed())
|
53
|
+
It("encodes", func() {
|
54
|
+
Expect(writeMulti(plain, 10, time.Time{})).To(Succeed())
|
55
|
+
Expect(writeMulti(compressed, 10, mockTime)).To(Succeed())
|
56
56
|
|
57
57
|
info, err := plain.Head(ctx)
|
58
58
|
Expect(err).NotTo(HaveOccurred())
|
59
59
|
Expect(info.Size).To(BeNumerically("~", 370, 10))
|
60
|
-
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "
|
60
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "0"}))
|
61
61
|
|
62
62
|
info, err = compressed.Head(ctx)
|
63
63
|
Expect(err).NotTo(HaveOccurred())
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.
|
4
|
+
version: 0.12.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-06-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -146,6 +146,7 @@ files:
|
|
146
146
|
- ext/parquet/types.go
|
147
147
|
- feedx.gemspec
|
148
148
|
- feedx.go
|
149
|
+
- feedx_ext_test.go
|
149
150
|
- feedx_test.go
|
150
151
|
- format.go
|
151
152
|
- format_test.go
|