pubsub_dumper 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9edf5816dfd8b5faea5275ff41b836f288433c62e4852564593e7fabf199d7c5
4
+ data.tar.gz: 67b0ba7f678280f9b5b810eb14d054561b79e7092e78b2684dcb6793740f8c1b
5
+ SHA512:
6
+ metadata.gz: 166cabc91cececb22ad251d09c4ed143e36392eb521fd9b13a9b79b056a4dcb59170905883632b8611f30c31cf9309c1d639443a6285e3e7c16ebf4448b38b68
7
+ data.tar.gz: 48b90c80d2f7c5964306a20be0faf8852bb7150c38839a27514ec03636bd3109661a749f3e03488ceae7cd4ca3da408751ab2ea1ec9a5b84cf3bc1b9ff201ecc
data/.gitignore ADDED
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.6.0
7
+ before_install: gem install bundler -v 1.17.2
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+
3
+ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
4
+ # Specify your gem's dependencies in pubsub_dumper.gemspec
5
+ gemspec
data/README.md ADDED
@@ -0,0 +1,42 @@
1
+ # PubsubDumper
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pubsub_dumper`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'pubsub_dumper'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install pubsub_dumper
22
+
23
+ ## Usage
24
+
25
+ ```bash
26
+ pubsub_dumper topic temporary-topic
27
+ pubsub_dumper topic temporary-topic 2019-05-14
28
+ pubsub_dumper topic temporary-topic 2019-05-14T15:30 2019-05-14T15:45
29
+
30
+ ```
31
+
32
+ TODO: Write usage instructions here
33
+
34
+ ## Development
35
+
36
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
37
+
38
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
39
+
40
+ ## Contributing
41
+
42
+ Bug reports and pull requests are welcome on GitHub at https://github.com/Fonsan/pubsub_dumper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task default: :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require 'pubsub_dumper'
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require 'irb'
14
+ IRB.start(__FILE__)
data/bin/pubsub_dumper ADDED
@@ -0,0 +1,46 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'google/cloud/pubsub'
4
+
5
+ pubsub = Google::Cloud::PubSub.new
6
+
7
+ topic_name, subscription_name, start, stop = ARGV
8
+ abort 'usage: pubsub_dumper topic subscription [start-utc] [stop-utc]' unless ARGV.size >= 2
9
+
10
+ topic = pubsub.topic(topic_name)
11
+ abort 'no such topic' unless topic
12
+ subscription = topic.subscription(subscription_name)
13
+ subscription ||= topic.subscribe(subscription_name, retain_acked: true)
14
+
15
+ start_time = start && Time.utc(*start.scan(/\d+/).map(&:to_i))
16
+ start_time ||= Time.new - 604800
17
+ stop_time = stop && Time.utc(*stop.scan(/\d+/).map(&:to_i))
18
+ subscription.seek(start_time)
19
+
20
+ concurrency = 10
21
+
22
+ queue = Queue.new
23
+
24
+ threads = concurrency.times.map do
25
+ Thread.start do
26
+ loop do
27
+ messages = subscription.pull(max: 1000)
28
+ in_range_messages, out_range_messages = messages.partition do |message|
29
+ !(stop_time && message.publish_time >= stop_time)
30
+ end
31
+ queue << in_range_messages.map(&:data).join
32
+ subscription.ack(in_range_messages)
33
+ out_range_messages.each(&:nack!)
34
+ break if out_range_messages.any? || messages.empty?
35
+ end
36
+ ensure
37
+ queue << nil
38
+ end
39
+ end
40
+
41
+ concurrency.times do
42
+ while buffer = queue.pop
43
+ STDOUT.write(buffer)
44
+ end
45
+ end
46
+ STDOUT.flush
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,3 @@
1
+ module PubsubDumper
2
+ VERSION = '0.1.0'.freeze
3
+ end
@@ -0,0 +1,6 @@
1
+ require 'pubsub_dumper/version'
2
+
3
+ module PubsubDumper
4
+ class Error < StandardError; end
5
+ # Your code goes here...
6
+ end
@@ -0,0 +1,26 @@
1
+ lib = File.expand_path('lib', __dir__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require 'pubsub_dumper/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'pubsub_dumper'
7
+ spec.version = PubsubDumper::VERSION
8
+ spec.authors = ['Fonsan']
9
+ spec.email = ['fonsan@gmail.com']
10
+
11
+ spec.summary = 'pubsub_dumper'
12
+ spec.description = 'pubsub_dumper'
13
+
14
+ # Specify which files should be added to the gem when it is released.
15
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
16
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
17
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ end
19
+ spec.bindir = 'bin'
20
+ spec.executables = 'pubsub_dumper'
21
+ spec.require_paths = ['lib']
22
+
23
+ spec.add_dependency 'google-cloud-pubsub', '0.35.0'
24
+ spec.add_development_dependency 'bundler', '~> 1.17'
25
+ spec.add_development_dependency 'rake', '~> 10.0'
26
+ end
metadata ADDED
@@ -0,0 +1,97 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: pubsub_dumper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Fonsan
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-05-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: google-cloud-pubsub
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '='
18
+ - !ruby/object:Gem::Version
19
+ version: 0.35.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '='
25
+ - !ruby/object:Gem::Version
26
+ version: 0.35.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.17'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.17'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ description: pubsub_dumper
56
+ email:
57
+ - fonsan@gmail.com
58
+ executables:
59
+ - pubsub_dumper
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - ".rspec"
65
+ - ".travis.yml"
66
+ - Gemfile
67
+ - README.md
68
+ - Rakefile
69
+ - bin/console
70
+ - bin/pubsub_dumper
71
+ - bin/setup
72
+ - lib/pubsub_dumper.rb
73
+ - lib/pubsub_dumper/version.rb
74
+ - pubsub_dumper.gemspec
75
+ homepage:
76
+ licenses: []
77
+ metadata: {}
78
+ post_install_message:
79
+ rdoc_options: []
80
+ require_paths:
81
+ - lib
82
+ required_ruby_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ required_rubygems_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ requirements: []
93
+ rubygems_version: 3.0.1
94
+ signing_key:
95
+ specification_version: 4
96
+ summary: pubsub_dumper
97
+ test_files: []