magellan-gcs-proxy 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.commit_template +40 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/magellan-gcs-proxy +5 -0
- data/lib/magellan/gcs/proxy.rb +14 -0
- data/lib/magellan/gcs/proxy/cli.rb +76 -0
- data/lib/magellan/gcs/proxy/context.rb +124 -0
- data/lib/magellan/gcs/proxy/expand_variable.rb +64 -0
- data/lib/magellan/gcs/proxy/file_operation.rb +46 -0
- data/lib/magellan/gcs/proxy/message_wrapper.rb +52 -0
- data/lib/magellan/gcs/proxy/pubsub_operation.rb +38 -0
- data/lib/magellan/gcs/proxy/version.rb +7 -0
- data/magellan-gcs-proxy.gemspec +30 -0
- metadata +136 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bca7eb4b15232e1190e59339f68cca01ef9929eb
|
4
|
+
data.tar.gz: c1d15432f65404b91aabae522b18ce2cac1fa172
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 59fa94585e7f984de73fd5d1cd36f5f499dcfa252e9def885effb9881fca87c4e7dd84d703d22d500c3358918216a0108c152e132ba93528507cce11f4b3736f
|
7
|
+
data.tar.gz: e27613b66bee86d93148171851cc19e0ec0a7a7a316d3600f457f6a9150a8660db8956bd064dade9c259bcf3d0bcbf6aa18f9fc8ad0d712ace7082e6d16e310c
|
data/.commit_template
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
# ==== Emojis ====
|
4
|
+
# ✏️ :pencil2: ドキュメント追加
|
5
|
+
# ð :bug: バグ修正
|
6
|
+
# ð :+1: 機能改善
|
7
|
+
# ✨ :sparkles: 部分的な機能追加
|
8
|
+
# ð :tada: 盛大に祝うべき大きな機能追加
|
9
|
+
# ♻️ :recycle: リファクタリング
|
10
|
+
# ð :shower: 不要な機能・使われなくなった機能の削除
|
11
|
+
# ð :green_heart: テストやCIの修正・改善
|
12
|
+
# ð :shirt: Lintエラーの修正やコードスタイルの修正
|
13
|
+
# ð :rocket: パフォーマンス改善
|
14
|
+
# ð :up: 依存パッケージなどのアップデート
|
15
|
+
# ð :lock: 新機能の公開範囲の制限
|
16
|
+
# ð :cop: セキュリティ関連の改善
|
17
|
+
|
18
|
+
|
19
|
+
# ==== Format ====
|
20
|
+
# :emoji: Subject
|
21
|
+
#
|
22
|
+
# Commit body...
|
23
|
+
|
24
|
+
# ==== The Seven Rules ====
|
25
|
+
# 1. Separate subject from body with a blank line
|
26
|
+
# 2. Limit the subject line to 50 characters
|
27
|
+
# 3. Capitalize the subject line
|
28
|
+
# 4. Do not end the subject line with a period
|
29
|
+
# 5. Use the imperative mood in the subject line
|
30
|
+
# 6. Wrap the body at 72 characters
|
31
|
+
# 7. Use the body to explain what and why vs. how
|
32
|
+
#
|
33
|
+
# How to Write a Git Commit Message http://chris.beams.io/posts/git-commit/
|
34
|
+
|
35
|
+
# for http://memo.goodpatch.co/2016/07/beautiful-commits-with-emojis/
|
36
|
+
|
37
|
+
# ==== Setup ====
|
38
|
+
#
|
39
|
+
# git config commit.template .commit_template
|
40
|
+
#
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
2
|
+
|
3
|
+
## Our Pledge
|
4
|
+
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
10
|
+
orientation.
|
11
|
+
|
12
|
+
## Our Standards
|
13
|
+
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
15
|
+
include:
|
16
|
+
|
17
|
+
* Using welcoming and inclusive language
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
19
|
+
* Gracefully accepting constructive criticism
|
20
|
+
* Focusing on what is best for the community
|
21
|
+
* Showing empathy towards other community members
|
22
|
+
|
23
|
+
Examples of unacceptable behavior by participants include:
|
24
|
+
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
26
|
+
advances
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
28
|
+
* Public or private harassment
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
30
|
+
address, without explicit permission
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
32
|
+
professional setting
|
33
|
+
|
34
|
+
## Our Responsibilities
|
35
|
+
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
38
|
+
response to any instances of unacceptable behavior.
|
39
|
+
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
44
|
+
threatening, offensive, or harmful.
|
45
|
+
|
46
|
+
## Scope
|
47
|
+
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
49
|
+
when an individual is representing the project or its community. Examples of
|
50
|
+
representing a project or community include using an official project e-mail
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
53
|
+
further defined and clarified by project maintainers.
|
54
|
+
|
55
|
+
## Enforcement
|
56
|
+
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
58
|
+
reported by contacting the project team at akm2000@gmail.com. All
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
63
|
+
|
64
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
65
|
+
faith may face temporary or permanent repercussions as determined by other
|
66
|
+
members of the project's leadership.
|
67
|
+
|
68
|
+
## Attribution
|
69
|
+
|
70
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
71
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
72
|
+
|
73
|
+
[homepage]: http://contributor-covenant.org
|
74
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 akm
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# Magellan::Gcs::Proxy
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/magellan/gcs/proxy`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'magellan-gcs-proxy'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install magellan-gcs-proxy
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/magellan-gcs-proxy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
36
|
+
|
37
|
+
|
38
|
+
## License
|
39
|
+
|
40
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "magellan/gcs/proxy"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require "magellan/gcs/proxy/version"
|
2
|
+
require 'magellan/gcs/proxy/cli'
|
3
|
+
require 'magellan/gcs/proxy/context'
|
4
|
+
require 'magellan/gcs/proxy/expand_variable'
|
5
|
+
require 'magellan/gcs/proxy/file_operation'
|
6
|
+
require 'magellan/gcs/proxy/message_wrapper'
|
7
|
+
require 'magellan/gcs/proxy/pubsub_operation'
|
8
|
+
|
9
|
+
module Magellan
|
10
|
+
module Gcs
|
11
|
+
module Proxy
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
require "magellan/gcs/proxy/pubsub_operation"
|
3
|
+
|
4
|
+
require 'json'
|
5
|
+
require 'logger'
|
6
|
+
require 'tmpdir'
|
7
|
+
|
8
|
+
module Magellan
|
9
|
+
module Gcs
|
10
|
+
module Proxy
|
11
|
+
class Cli
|
12
|
+
include PubsubOperation
|
13
|
+
|
14
|
+
attr_reader :cmd_template
|
15
|
+
def initialize(*args)
|
16
|
+
@cmd_template = args.join(' ')
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
logger.info("Start listening")
|
21
|
+
sub.listen do |msg|
|
22
|
+
begin
|
23
|
+
process(msg)
|
24
|
+
rescue => e
|
25
|
+
logger.error("[#{e.class.name}] #{e.message}")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
rescue => e
|
29
|
+
logger.error("[#{e.class.name}] #{e.message}")
|
30
|
+
raise e
|
31
|
+
end
|
32
|
+
|
33
|
+
def process(msg)
|
34
|
+
logger.info("Processing message: #{msg.inspect}")
|
35
|
+
Dir.mktmpdir 'workspace' do |dir|
|
36
|
+
dfiles = parse(msg.attributes['download_files'])
|
37
|
+
ufiles = parse(msg.attributes['upload_files'])
|
38
|
+
|
39
|
+
context = Context.new(dir, dfiles, ufiles)
|
40
|
+
context.setup
|
41
|
+
|
42
|
+
context.download
|
43
|
+
|
44
|
+
cmd = build_command(msg, context)
|
45
|
+
|
46
|
+
logger.info("Executing command: #{cmd.inspect}")
|
47
|
+
|
48
|
+
if system(cmd)
|
49
|
+
context.upload
|
50
|
+
|
51
|
+
sub.acknowledge msg
|
52
|
+
logger.info("Complete processing and acknowledged")
|
53
|
+
else
|
54
|
+
logger.error("Error: #{cmd.inspect}")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def logger
|
60
|
+
@logger ||= Logger.new($stdout)
|
61
|
+
end
|
62
|
+
|
63
|
+
def parse(str)
|
64
|
+
return nil if str.nil? || str.empty?
|
65
|
+
JSON.parse(str)
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
def build_command(msg, context)
|
70
|
+
msg_wrapper = MessageWrapper.new(msg, context)
|
71
|
+
ExpandVariable.expand_variables(cmd_template, msg_wrapper)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
require "magellan/gcs/proxy/file_operation"
|
3
|
+
|
4
|
+
require 'uri'
|
5
|
+
|
6
|
+
module Magellan
|
7
|
+
module Gcs
|
8
|
+
module Proxy
|
9
|
+
class Context
|
10
|
+
include FileOperation
|
11
|
+
|
12
|
+
attr_reader :workspace, :remote_download_files, :remote_upload_files
|
13
|
+
def initialize(workspace, remote_download_files, remote_upload_files)
|
14
|
+
@workspace = workspace
|
15
|
+
@remote_download_files = remote_download_files
|
16
|
+
@remote_upload_files = remote_upload_files
|
17
|
+
end
|
18
|
+
|
19
|
+
KEYS = [
|
20
|
+
:workspace,
|
21
|
+
:downloads_dir, :uploads_dir,
|
22
|
+
:download_files, :upload_files,
|
23
|
+
:local_download_files, :local_upload_files,
|
24
|
+
:remote_download_files, :remote_upload_files
|
25
|
+
].freeze
|
26
|
+
|
27
|
+
def [](key)
|
28
|
+
case key.to_sym
|
29
|
+
when *KEYS then send(key)
|
30
|
+
else nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def include?(key)
|
35
|
+
KEYS.include?(key)
|
36
|
+
end
|
37
|
+
|
38
|
+
def downloads_dir
|
39
|
+
File.join(workspace, 'downloads')
|
40
|
+
end
|
41
|
+
|
42
|
+
def download_mapping
|
43
|
+
@download_mapping ||= build_mapping(downloads_dir, remote_download_files)
|
44
|
+
end
|
45
|
+
|
46
|
+
def local_download_files
|
47
|
+
@local_download_files ||= build_local_files_obj(remote_download_files, download_mapping)
|
48
|
+
end
|
49
|
+
alias_method :download_files, :local_download_files
|
50
|
+
|
51
|
+
def uploads_dir
|
52
|
+
File.join(workspace, 'uploads')
|
53
|
+
end
|
54
|
+
|
55
|
+
def upload_mapping
|
56
|
+
@upload_mapping ||= build_mapping(uploads_dir, remote_upload_files)
|
57
|
+
end
|
58
|
+
|
59
|
+
def local_upload_files
|
60
|
+
@local_upload_files ||= build_local_files_obj(remote_upload_files, upload_mapping)
|
61
|
+
end
|
62
|
+
alias_method :upload_files, :local_upload_files
|
63
|
+
|
64
|
+
def setup
|
65
|
+
setup_dirs
|
66
|
+
end
|
67
|
+
|
68
|
+
def download
|
69
|
+
download_mapping.each do |url, path|
|
70
|
+
logger.info("Downloading: #{url}")
|
71
|
+
uri = parse_uri(url)
|
72
|
+
bucket = storage.bucket(uri.host)
|
73
|
+
file = bucket.file uri.path.sub(/\A\//, '')
|
74
|
+
file.download(path)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def upload
|
79
|
+
upload_mapping.each do |url, path|
|
80
|
+
logger.info("Uploading: #{url}")
|
81
|
+
uri = parse_uri(url)
|
82
|
+
bucket = storage.bucket(uri.host)
|
83
|
+
bucket.create_file path, uri.path.sub(/\A\//, '')
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def setup_dirs
|
88
|
+
[:downloads_dir, :uploads_dir].each{|k| Dir.mkdir(send(k))}
|
89
|
+
end
|
90
|
+
|
91
|
+
def build_mapping(base_dir, obj)
|
92
|
+
flatten_values(obj).flatten.each_with_object({}) do |url, d|
|
93
|
+
uri = parse_uri(url)
|
94
|
+
d[url] = File.join(base_dir, uri.path)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def flatten_values(obj)
|
99
|
+
case obj
|
100
|
+
when Hash then flatten_values(obj.values)
|
101
|
+
when Array then obj.map{|i| flatten_values(i) }
|
102
|
+
else obj
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def parse_uri(str)
|
107
|
+
uri = URI.parse(str)
|
108
|
+
raise "Unsupported scheme #{uri.scheme.inspect} of #{str}" unless uri.scheme == 'gs'
|
109
|
+
uri
|
110
|
+
end
|
111
|
+
|
112
|
+
def build_local_files_obj(obj, mapping)
|
113
|
+
case obj
|
114
|
+
when Hash then obj.each_with_object({}){|(k,v), d| d[k] = build_local_files_obj(v, mapping)}
|
115
|
+
when Array then obj.map{|i| build_local_files_obj(i, mapping)}
|
116
|
+
when String then mapping[obj]
|
117
|
+
else obj
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
|
3
|
+
module Magellan
|
4
|
+
module Gcs
|
5
|
+
module Proxy
|
6
|
+
module ExpandVariable
|
7
|
+
|
8
|
+
class InvalidReferenceError < StandardError
|
9
|
+
end
|
10
|
+
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def dig_variables(variable_ref, data)
|
14
|
+
vars = variable_ref.split(".").map{|i| (/\A\d+\z/.match(i)) ? i.to_i : i }
|
15
|
+
value = vars.inject(data) do |tmp, v|
|
16
|
+
case v
|
17
|
+
when String
|
18
|
+
if tmp.respond_to?(:[]) && tmp.respond_to?(:include?)
|
19
|
+
if tmp.include?(v)
|
20
|
+
tmp[v]
|
21
|
+
else
|
22
|
+
raise InvalidReferenceError, variable_ref
|
23
|
+
end
|
24
|
+
else
|
25
|
+
raise InvalidReferenceError, variable_ref
|
26
|
+
end
|
27
|
+
when Integer
|
28
|
+
case tmp
|
29
|
+
when Array
|
30
|
+
if tmp.size > v
|
31
|
+
tmp[v]
|
32
|
+
else
|
33
|
+
raise InvalidReferenceError, variable_ref
|
34
|
+
end
|
35
|
+
else
|
36
|
+
raise InvalidReferenceError, variable_ref
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def expand_variables(str, data, quote_string: false)
|
43
|
+
data ||= {}
|
44
|
+
str.gsub(/\%\{\s*([\w.]+)\s*\}/) do |m|
|
45
|
+
var = Regexp.last_match(1)
|
46
|
+
value =
|
47
|
+
begin
|
48
|
+
dig_variables(var, data)
|
49
|
+
rescue InvalidReferenceError
|
50
|
+
""
|
51
|
+
end
|
52
|
+
|
53
|
+
case value
|
54
|
+
when String then quote_string ? value.to_s : value
|
55
|
+
when Array then value.flatten.join(' ')
|
56
|
+
else value.to_s
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
|
3
|
+
require 'uri'
|
4
|
+
require "google/cloud/storage"
|
5
|
+
|
6
|
+
module Magellan
|
7
|
+
module Gcs
|
8
|
+
module Proxy
|
9
|
+
module FileOperation
|
10
|
+
def storage
|
11
|
+
@storage ||= Google::Cloud::Storage.new(
|
12
|
+
# default credential を利用するため、プロジェクトの指定はしない
|
13
|
+
# project: ENV['GOOGLE_PROJECT'] || 'dummy-project-id',
|
14
|
+
# keyfile: ENV['GOOGLE_KEY_JSON_FILE'],
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
def download(base_dir, urls)
|
19
|
+
(urls || []).each do |url|
|
20
|
+
logger.info("Downloading: #{url}")
|
21
|
+
uri = parse_uri(url)
|
22
|
+
bucket = storage.bucket(uri.host)
|
23
|
+
file = bucket.file uri.path.sub(/\A\//, '')
|
24
|
+
file.download File.join(base_dir, uri.path)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def upload(base_dir, urls)
|
29
|
+
(urls || []).each do |url|
|
30
|
+
logger.info("Uploading: #{url}")
|
31
|
+
uri = parse_uri(url)
|
32
|
+
bucket = storage.bucket(uri.host)
|
33
|
+
bucket.create_file File.join(base_dir, uri.path), uri.path.sub(/\A\//, '')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def parse_uri(str)
|
38
|
+
uri = URI.parse(str)
|
39
|
+
raise "Unsupported scheme #{uri.scheme.inspect} of #{str}" unless uri.scheme == 'gs'
|
40
|
+
uri
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
|
3
|
+
module Magellan
|
4
|
+
module Gcs
|
5
|
+
module Proxy
|
6
|
+
class MessageWrapper
|
7
|
+
attr_reader :msg, :context
|
8
|
+
def initialize(msg, context)
|
9
|
+
@msg, @context = msg, context
|
10
|
+
end
|
11
|
+
|
12
|
+
def [](key)
|
13
|
+
case key.to_sym
|
14
|
+
when :attrs, :attributes then return attributes
|
15
|
+
when :data then return msg.data
|
16
|
+
end
|
17
|
+
context[key.to_sym]
|
18
|
+
end
|
19
|
+
|
20
|
+
def include?(key)
|
21
|
+
k = key.to_sym
|
22
|
+
context.include?(k) || [:attrs, :attributes, :data].include?(k)
|
23
|
+
end
|
24
|
+
|
25
|
+
def attributes
|
26
|
+
Attrs.new(msg.attributes)
|
27
|
+
end
|
28
|
+
|
29
|
+
class Attrs
|
30
|
+
attr_reader :data
|
31
|
+
def initialize(data)
|
32
|
+
@data = data
|
33
|
+
end
|
34
|
+
|
35
|
+
def [](key)
|
36
|
+
value = data[key]
|
37
|
+
if value.is_a?(String) && value =~ /\A\[.*\]\z|\A\{.*\}\z/
|
38
|
+
JSON.parse(value) rescue value
|
39
|
+
else
|
40
|
+
value
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def include?(key)
|
45
|
+
data.include?(key) || data.include?(key.to_sym)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require "magellan/gcs/proxy"
|
2
|
+
|
3
|
+
require "google/cloud/pubsub"
|
4
|
+
|
5
|
+
module Magellan
|
6
|
+
module Gcs
|
7
|
+
module Proxy
|
8
|
+
module PubsubOperation
|
9
|
+
def pubsub
|
10
|
+
@pubsub ||= Google::Cloud::Pubsub.new(
|
11
|
+
# default credential を利用するため、プロジェクトの指定はしない
|
12
|
+
# project: ENV['GOOGLE_PROJECT'] || 'dummy-project-id',
|
13
|
+
# keyfile: ENV['GOOGLE_KEY_JSON_FILE'],
|
14
|
+
)
|
15
|
+
end
|
16
|
+
|
17
|
+
def topic
|
18
|
+
unless @topic
|
19
|
+
topic_name = ENV['BATCH_TOPIC_NAME'] || 'test-topic'
|
20
|
+
@topic = pubsub.topic(topic_name) || pubsub.create_topic(topic_name)
|
21
|
+
logger.info("topic: #{@topic.inspect}")
|
22
|
+
end
|
23
|
+
@topic
|
24
|
+
end
|
25
|
+
|
26
|
+
def sub
|
27
|
+
unless @sub
|
28
|
+
sub_name = ENV['BATCH_SUBSCRIPTION_NAME'] || 'test-subscription'
|
29
|
+
@sub = topic.subscription(sub_name) || topic.subscribe(sub_name)
|
30
|
+
logger.info("subscription: #{@sub.inspect}")
|
31
|
+
end
|
32
|
+
@sub
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'magellan/gcs/proxy/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "magellan-gcs-proxy"
|
8
|
+
spec.version = Magellan::Gcs::Proxy::VERSION
|
9
|
+
spec.authors = ["akm"]
|
10
|
+
spec.email = ["akm2000@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Adaptor for MAGELLAN BLOCKS batch type IoT board}
|
13
|
+
spec.description = %q{Adaptor for MAGELLAN BLOCKS batch type IoT board}
|
14
|
+
spec.homepage = "https://github.com/groovenauts/magellan-gcs-proxy"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_runtime_dependency "google-cloud-pubsub"
|
25
|
+
spec.add_runtime_dependency "google-cloud-storage"
|
26
|
+
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.13"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
30
|
+
end
|
metadata
ADDED
@@ -0,0 +1,136 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: magellan-gcs-proxy
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- akm
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-11-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: google-cloud-pubsub
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: google-cloud-storage
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.13'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.13'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '10.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '10.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.0'
|
83
|
+
description: Adaptor for MAGELLAN BLOCKS batch type IoT board
|
84
|
+
email:
|
85
|
+
- akm2000@gmail.com
|
86
|
+
executables:
|
87
|
+
- magellan-gcs-proxy
|
88
|
+
extensions: []
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- ".commit_template"
|
92
|
+
- ".gitignore"
|
93
|
+
- ".rspec"
|
94
|
+
- ".travis.yml"
|
95
|
+
- CODE_OF_CONDUCT.md
|
96
|
+
- Gemfile
|
97
|
+
- LICENSE.txt
|
98
|
+
- README.md
|
99
|
+
- Rakefile
|
100
|
+
- bin/console
|
101
|
+
- bin/setup
|
102
|
+
- exe/magellan-gcs-proxy
|
103
|
+
- lib/magellan/gcs/proxy.rb
|
104
|
+
- lib/magellan/gcs/proxy/cli.rb
|
105
|
+
- lib/magellan/gcs/proxy/context.rb
|
106
|
+
- lib/magellan/gcs/proxy/expand_variable.rb
|
107
|
+
- lib/magellan/gcs/proxy/file_operation.rb
|
108
|
+
- lib/magellan/gcs/proxy/message_wrapper.rb
|
109
|
+
- lib/magellan/gcs/proxy/pubsub_operation.rb
|
110
|
+
- lib/magellan/gcs/proxy/version.rb
|
111
|
+
- magellan-gcs-proxy.gemspec
|
112
|
+
homepage: https://github.com/groovenauts/magellan-gcs-proxy
|
113
|
+
licenses:
|
114
|
+
- MIT
|
115
|
+
metadata: {}
|
116
|
+
post_install_message:
|
117
|
+
rdoc_options: []
|
118
|
+
require_paths:
|
119
|
+
- lib
|
120
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '0'
|
130
|
+
requirements: []
|
131
|
+
rubyforge_project:
|
132
|
+
rubygems_version: 2.5.1
|
133
|
+
signing_key:
|
134
|
+
specification_version: 4
|
135
|
+
summary: Adaptor for MAGELLAN BLOCKS batch type IoT board
|
136
|
+
test_files: []
|