sideload 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d48ebf35a1d216a2eba4ade7f65167af42af996e
4
+ data.tar.gz: c87f65e73d1112165856ca4cfd4aea432521a1c3
5
+ SHA512:
6
+ metadata.gz: 63f31c1154cef0c30b595840939018ca3dc505acb924fd09be384fd78d99303815642803ab461187e1ede1d906281564a20cff478c4cbdc5573cc2c477e544f4
7
+ data.tar.gz: 5b504063d8583fcf30c112a8fbc9445c027120f9d09ec721d466df10e0b28c4401d59f48ed93a93c3753c36cb73606b7801b6db4f0d20711558319fb5bcf9c02
data/LICENSE ADDED
@@ -0,0 +1,25 @@
1
+ BSD 2-Clause License
2
+
3
+ Copyright (c) 2017, Matthias Geier
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ * Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ * Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
20
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
data/lib/sideload.rb ADDED
@@ -0,0 +1,51 @@
1
+ require "redis"
2
+ require "sideload/validation_error"
3
+ require "sideload/config"
4
+ require "sideload/redis"
5
+ require "sideload/path"
6
+ require "sideload/github"
7
+
8
+ module Sideload
9
+ extend self
10
+
11
+ def logger=(logger)
12
+ @logger = logger
13
+ end
14
+
15
+ def init
16
+ c = Config.new
17
+ yield(c)
18
+ return c
19
+ end
20
+
21
+ def update!(sources)
22
+ scope, arg, _config, validate = sources.shift
23
+ mod = const_get(scope.to_s.capitalize)
24
+ contents = mod.read(*arg)
25
+ unless sources.empty?
26
+ next_layer = update!(sources)
27
+ unless next_layer.nil?
28
+ (contents.keys | next_layer.keys).each do |key|
29
+ if !next_layer.has_key?(key)
30
+ mod.with(arg, key) do |fp, t|
31
+ mod.delete(fp, t)
32
+ end
33
+ else
34
+ mod.with(arg, key) do |fp, t|
35
+ mod.write(fp, t, next_layer[key], &validate)
36
+ end
37
+ end
38
+ end
39
+ contents = mod.read(arg)
40
+ end
41
+ end
42
+ contents.each(&Proc.new) if block_given?
43
+ return contents
44
+ rescue ValidationError => e
45
+ if @logger
46
+ @logger.error { [e.class, e.message].inspect }
47
+ @logger.debug { e.backtrace.join("\n") }
48
+ end
49
+ return nil
50
+ end
51
+ end
@@ -0,0 +1,41 @@
1
+ module Sideload
2
+ class Config
3
+ ALLOWED = %i[path github redis]
4
+
5
+ attr_reader :sources, :packer, :unpacker
6
+
7
+ def initialize
8
+ @sources = []
9
+ @packer = ->(f) { f }
10
+ @unpacker = ->(f) { f }
11
+ end
12
+
13
+ def pack
14
+ @packer = Proc.new
15
+ end
16
+
17
+ def unpack
18
+ @unpacker = Proc.new
19
+ end
20
+
21
+ def validate
22
+ @validate = Proc.new
23
+ end
24
+
25
+ def source(scope, arg, **config)
26
+ if !ALLOWED.include?(scope)
27
+ raise "scope #{scope.inspect} not in #{ALLOWED.inspect}"
28
+ end
29
+ @sources << [
30
+ scope,
31
+ arg,
32
+ config,
33
+ (block_given? ? Proc.new : nil) || @validate
34
+ ]
35
+ end
36
+
37
+ def update!
38
+ Sideload.update!(@sources.dup, &(block_given? ? Proc.new : nil))
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,91 @@
1
+ require "net/http"
2
+ require "base64"
3
+ require "json"
4
+
5
+ module Sideload
6
+ module Github
7
+ extend self
8
+
9
+ GITHUB_V3 = URI("https://api.github.com")
10
+
11
+ def credentials=(arr)
12
+ @user, @pass = arr
13
+ end
14
+
15
+ def read(repo, path)
16
+ sha = navigate_to(repo, path)
17
+ return traverse(repo, sha)
18
+ end
19
+
20
+ def with(path, fname)
21
+ raise RuntimeError.new("not implemented")
22
+ end
23
+
24
+ def write(full_path, target, content)
25
+ raise RuntimeError.new("not implemented")
26
+ end
27
+
28
+ def delete(full_path, target)
29
+ raise RuntimeError.new("not implemented")
30
+ end
31
+
32
+ private
33
+
34
+ def cred!
35
+ raise RuntimeError.new("no github credentials set") if @user.nil?
36
+ end
37
+
38
+ def traverse(repo, sha, path = [])
39
+ return get_tree(repo, sha).reduce({}) do |acc, node|
40
+ name = node["path"]
41
+ case node["type"]
42
+ when "blob"
43
+ acc[(path + [name]).join("/")] = get_blob(node["url"])
44
+ when "tree"
45
+ acc.merge!(traverse(repo, node["sha"], path + [name]))
46
+ end
47
+ next acc
48
+ end
49
+ end
50
+
51
+ def navigate_to(repo, path)
52
+ return path.split("/").reduce("master") do |acc, folder|
53
+ next acc if folder.empty?
54
+ get_tree(repo, acc)&.detect { |e| e["type"] && e["path"] == folder }&.
55
+ []("sha")
56
+ end
57
+ end
58
+
59
+ def get_blob(url)
60
+ cred!
61
+ uri = URI(url)
62
+ http = Net::HTTP.new(uri.host, uri.port)
63
+ http.use_ssl = uri.scheme == "https"
64
+ request = Net::HTTP::Get.new(uri.request_uri)
65
+ request.basic_auth(@user, @pass)
66
+ response = http.request(request)
67
+ if response.is_a?(Net::HTTPOK)
68
+ return Base64.decode64(JSON.parse(response.body)&.[]("content"))
69
+ else
70
+ puts response, url
71
+ return nil
72
+ end
73
+ end
74
+
75
+ def get_tree(repo, folder)
76
+ cred!
77
+ uri = URI.join(GITHUB_V3, "/repos/#{repo}/git/trees/#{folder}")
78
+ http = Net::HTTP.new(uri.host, uri.port)
79
+ http.use_ssl = uri.scheme == "https"
80
+ request = Net::HTTP::Get.new(uri.request_uri)
81
+ request.basic_auth(@user, @pass)
82
+ response = http.request(request)
83
+ if response.is_a?(Net::HTTPOK)
84
+ return JSON.parse(response.body)&.[]("tree")
85
+ else
86
+ puts response, uri
87
+ return nil
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,39 @@
1
+ module Sideload
2
+ module Path
3
+ extend self
4
+
5
+ def read(path)
6
+ dir = File.expand_path(path)
7
+ return Dir[dir + "/**/*.*"].map do |fname|
8
+ [fname.sub(dir + "/", ""), File.open(fname) { |f| f.read }]
9
+ end.to_h
10
+ end
11
+
12
+ def with(path, fname)
13
+ *dirs, target = fname.split("/")
14
+ full_path = dirs.reduce(File.expand_path(path)) do |acc, dir|
15
+ f = File.join(acc, dir)
16
+ Dir.mkdir(f) unless File.directory?(f)
17
+ next f
18
+ end
19
+ yield(full_path, target)
20
+ dirs.reverse.each do |dir|
21
+ break if File.basename(full_path) != dir
22
+ break unless Dir[full_path + "/*.*"].empty?
23
+ Dir.rmdir(full_path)
24
+ full_path = File.dirname(full_path)
25
+ end
26
+ end
27
+
28
+ def write(full_path, target, content)
29
+ if block_given? && !yield(content)
30
+ raise ValidationError.new(self, "#{full_path}/#{target}", content)
31
+ end
32
+ File.open(File.join(full_path, target), "w") { |f| f.print content }
33
+ end
34
+
35
+ def delete(full_path, target)
36
+ File.delete(File.join(full_path, target))
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,34 @@
1
+ module Sideload
2
+ module Redis
3
+ extend self
4
+
5
+ def db!(**config)
6
+ @redis = ::Redis.new(**config)
7
+ end
8
+
9
+ def db
10
+ @redis || db!
11
+ end
12
+
13
+ def read(path)
14
+ return db.keys(path + "*").map do |key|
15
+ [key.sub(path, ""), db.get(key)]
16
+ end.to_h
17
+ end
18
+
19
+ def with(path, fname)
20
+ yield(path, fname)
21
+ end
22
+
23
+ def write(full_path, target, content)
24
+ if block_given? && !yield(content)
25
+ raise ValidationError.new(self, "#{full_path}/#{target}", content)
26
+ end
27
+ db.set(File.join(full_path, target), content)
28
+ end
29
+
30
+ def delete(full_path, target)
31
+ db.del(File.join(full_path, target))
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,7 @@
1
+ module Sideload
2
+ class ValidationError < RuntimeError
3
+ def initialize(klass, path, content)
4
+ super("validation error in #{klass}: #{path}\n#{content}")
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,3 @@
1
+ module Sideload
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,107 @@
1
+ describe "Sideload::Config" do
2
+ before do
3
+ @config = Sideload::Config.new
4
+ end
5
+
6
+ after do
7
+ @config = nil
8
+ end
9
+
10
+ describe ".new" do
11
+ it "initializes with empty sources" do
12
+ assert_equal [], @config.sources
13
+ end
14
+
15
+ it "initializes with forwarding packer and unpacker" do
16
+ assert_equal 25, @config.packer.call(25)
17
+ assert_equal 25, @config.unpacker.call(25)
18
+ end
19
+ end
20
+
21
+ describe "#pack" do
22
+ it "overrides the packer with proc" do
23
+ blk = ->(f) { f * 2 }
24
+ assert_equal 25, @config.packer.call(25)
25
+ @config.pack(&blk)
26
+ assert_equal 50, @config.packer.call(25)
27
+ end
28
+ end
29
+
30
+ describe "#unpack" do
31
+ it "overrides the unpacker with proc" do
32
+ blk = ->(f) { f * 2 }
33
+ assert_equal 25, @config.unpacker.call(25)
34
+ @config.unpack(&blk)
35
+ assert_equal 50, @config.unpacker.call(25)
36
+ end
37
+ end
38
+
39
+ describe "#source" do
40
+ it "allows path, web and redis as scopes" do
41
+ %i[path github redis].each do |scope|
42
+ raised = false
43
+ begin
44
+ @config.source(scope, *["arg"])
45
+ rescue
46
+ raised = true
47
+ end
48
+ refute raised
49
+ end
50
+ end
51
+
52
+ it "fails on any other scope" do
53
+ raised = false
54
+ begin
55
+ @config.source(:nuu, "arg")
56
+ rescue
57
+ raised = true
58
+ end
59
+ assert raised
60
+ end
61
+
62
+ it "stores at the back of sources" do
63
+ @config.source(:path, "first/")
64
+ assert_equal [[:path, "first/", {}, nil]], @config.sources
65
+ @config.source(:path, "second/")
66
+ assert_equal [[:path, "first/", {}, nil], [:path, "second/", {}, nil]],
67
+ @config.sources
68
+ end
69
+
70
+ it "stores any config opts" do
71
+ @config.source(:path, "first/", any: "opt", with: 1)
72
+ assert_equal [[:path, "first/", {any: "opt", with: 1}, nil]],
73
+ @config.sources
74
+ end
75
+
76
+ it "uses a generic validate when available, otherwise given block" do
77
+ @config.validate { |f| f == 25 }
78
+ @config.source(:path, nil)
79
+ validator = @config.sources.last.last
80
+ assert validator.call(25)
81
+ refute validator.call(26)
82
+ @config.source(:path, nil) { |f| f == 26 }
83
+ validator = @config.sources.last.last
84
+ assert validator.call(26)
85
+ refute validator.call(25)
86
+ end
87
+ end
88
+
89
+ describe "#update!" do
90
+ it "forwards a block and the sources array to sideload" do
91
+ @config.source(:path, nil)
92
+ ran = 0
93
+ test_val = Proc.new do |sources|
94
+ assert_equal @config.sources, sources
95
+ ran += 1
96
+ end
97
+ Sideload.stub(:update!, test_val, 25) do
98
+ @config.update!
99
+ @config.update! do |f|
100
+ assert_equal 25, f
101
+ ran += 1
102
+ end
103
+ end
104
+ assert_equal 3, ran
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,33 @@
1
+ describe "Sideload::Github" do
2
+ after do
3
+ Sideload::Github.instance_variable_set("@user", nil)
4
+ Sideload::Github.instance_variable_set("@pass", nil)
5
+ end
6
+
7
+ describe ".credentials=" do
8
+ it "sets credentials for github" do
9
+ Sideload::Github.credentials = ["moo", "bar"]
10
+ assert_equal "moo", Sideload::Github.instance_variable_get("@user")
11
+ assert_equal "bar", Sideload::Github.instance_variable_get("@pass")
12
+ end
13
+ end
14
+
15
+ describe ".read" do
16
+ it "raises when no credentials are set" do
17
+ raised = false
18
+ begin
19
+ Sideload::Github.read("matthias-geier/sideload", "sample_sources")
20
+ rescue RuntimeError => e
21
+ assert_equal "no github credentials set", e.message
22
+ raised = true
23
+ end
24
+ assert raised
25
+ end
26
+
27
+ it "lists keys and contents under a path" do
28
+ Sideload::Github.credentials = ["e608c053ca8068b09a5fbc7f337cb22f11cf7725", "x-oauth-basic"]
29
+ assert_equal({"toot.json" => "[]\n"}, Sideload::Github.
30
+ read("matthias-geier/sideload", "sample_sources"))
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,62 @@
1
+ describe "Sideload::Path" do
2
+ describe ".read" do
3
+ it "reads files on a given path" do
4
+ assert_equal({"toot.json" => "[]\n"},
5
+ Sideload::Path.read("sample_sources/"))
6
+ end
7
+ end
8
+
9
+ describe ".with" do
10
+ it "creates and clears missing folders for the target" do
11
+ refute File.exist?("sample_sources/cookie")
12
+ Sideload::Path.with("sample_sources/", "cookie/joe.json") do |_path, file|
13
+ assert File.exist?("sample_sources/cookie")
14
+ assert_equal "joe.json", file
15
+ end
16
+ refute File.exist?("sample_sources/cookie")
17
+ end
18
+ end
19
+
20
+ describe ".write" do
21
+ it "validates against content with block" do
22
+ m = Module.new { def self.print(_); end }
23
+ File.stub(:open, nil, m) do
24
+ raised = false
25
+ ran = 0
26
+ begin
27
+ Sideload::Path.write("", "", "{}") do |c|
28
+ assert_equal "{}", c
29
+ ran += 1
30
+ true
31
+ end
32
+ Sideload::Path.write("", "", "{}") do |c|
33
+ assert_equal "{}", c
34
+ ran += 1
35
+ false
36
+ end
37
+ rescue Sideload::ValidationError
38
+ raised = true
39
+ end
40
+ assert raised
41
+ assert_equal 2, ran
42
+ end
43
+ end
44
+
45
+ it "writes content to a file" do
46
+ m = Module.new { def self.print(c); @ran = 1; @content = c; end }
47
+ File.stub(:open, nil, m) do
48
+ Sideload::Path.write("", "", "{}")
49
+ end
50
+ assert_equal 1, m.instance_variable_get("@ran")
51
+ assert_equal "{}", m.instance_variable_get("@content")
52
+ end
53
+ end
54
+
55
+ describe ".delete" do
56
+ it "deletes a file" do
57
+ File.stub(:delete, ->(path) { assert_equal "s/m.json", path }) do
58
+ Sideload::Path.delete("s", "m.json")
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,84 @@
1
+ describe "Sideload::Path" do
2
+ after do
3
+ db = Sideload::Redis.db
4
+ db.del("moo/bar", "blu/bar", "s/m.json", "x/m.json")
5
+ Sideload::Redis.remove_instance_variable("@redis")
6
+ end
7
+
8
+ describe ".db" do
9
+ it "initiaizes and delivers the same redis instance" do
10
+ assert Sideload::Redis.db.is_a?(Redis)
11
+ assert_equal Sideload::Redis.db, Sideload::Redis.db
12
+ end
13
+ end
14
+
15
+ describe ".db!" do
16
+ it "initializes a new redis instance with params" do
17
+ db = Sideload::Redis.db
18
+ Sideload::Redis.db!(db: 3)
19
+ assert Sideload::Redis.db.is_a?(Redis)
20
+ refute_equal db, Sideload::Redis.db
21
+ end
22
+ end
23
+
24
+ describe ".read" do
25
+ it "lists keys and contents under a path" do
26
+ db = Sideload::Redis.db
27
+ db.set("moo/bar", "{}")
28
+ db.set("blu/bar", "{}")
29
+ assert_equal({"bar" => "{}"}, Sideload::Redis.read("moo/"))
30
+ end
31
+ end
32
+
33
+ describe ".with" do
34
+ it "yields with given params" do
35
+ ran = false
36
+ Sideload::Redis.with(1, 2) do |path, fname|
37
+ ran = true
38
+ assert_equal 1, path
39
+ assert_equal 2, fname
40
+ end
41
+ assert ran
42
+ end
43
+ end
44
+
45
+ describe ".write" do
46
+ it "validates against content with block" do
47
+ raised = false
48
+ ran = 0
49
+ begin
50
+ Sideload::Redis.write("s", "m.json", "{}") do |c|
51
+ assert_equal "{}", c
52
+ ran += 1
53
+ true
54
+ end
55
+ Sideload::Redis.write("x", "m.json", "{}") do |c|
56
+ assert_equal "{}", c
57
+ ran += 1
58
+ false
59
+ end
60
+ rescue Sideload::ValidationError
61
+ raised = true
62
+ end
63
+ db = Sideload::Redis.db
64
+ assert_equal "{}", db.get("s/m.json")
65
+ refute db.exists("x/m.json")
66
+ assert raised
67
+ assert_equal 2, ran
68
+ end
69
+
70
+ it "writes content to key" do
71
+ Sideload::Redis.write("s", "m.json", "{}")
72
+ assert_equal "{}", Sideload::Redis.db.get("s/m.json")
73
+ end
74
+ end
75
+
76
+ describe ".delete" do
77
+ it "deletes a key" do
78
+ db = Sideload::Redis.db
79
+ db.set("s/m.json", "{}")
80
+ Sideload::Redis.delete("s", "m.json")
81
+ refute db.exists("s/m.json")
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,7 @@
1
+ require "sideload"
2
+ require "json"
3
+ require "minitest/autorun"
4
+
5
+ Dir["test/**/*_spec.rb"].each do |f|
6
+ load f
7
+ end
metadata ADDED
@@ -0,0 +1,75 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sideload
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Matthias Geier
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-10-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: minitest
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '5.10'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '5.10'
27
+ description: Offers a safe way to copy (config) files through differentstorages
28
+ email: 'mayutamano@gmail.com '
29
+ executables: []
30
+ extensions: []
31
+ extra_rdoc_files: []
32
+ files:
33
+ - LICENSE
34
+ - lib/sideload.rb
35
+ - lib/sideload/config.rb
36
+ - lib/sideload/github.rb
37
+ - lib/sideload/path.rb
38
+ - lib/sideload/redis.rb
39
+ - lib/sideload/validation_error.rb
40
+ - lib/sideload/version.rb
41
+ - test/sideload/config_spec.rb
42
+ - test/sideload/github_spec.rb
43
+ - test/sideload/path_spec.rb
44
+ - test/sideload/redis_spec.rb
45
+ - test/test_runner.rb
46
+ homepage: https://github.com/matthias-geier/sideload
47
+ licenses:
48
+ - BSD-2-Clause
49
+ metadata: {}
50
+ post_install_message:
51
+ rdoc_options: []
52
+ require_paths:
53
+ - lib
54
+ required_ruby_version: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - ">="
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ required_rubygems_version: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ requirements: []
65
+ rubyforge_project:
66
+ rubygems_version: 2.5.1
67
+ signing_key:
68
+ specification_version: 4
69
+ summary: Validated data copying
70
+ test_files:
71
+ - test/sideload/redis_spec.rb
72
+ - test/sideload/path_spec.rb
73
+ - test/sideload/github_spec.rb
74
+ - test/sideload/config_spec.rb
75
+ - test/test_runner.rb