sideload 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d48ebf35a1d216a2eba4ade7f65167af42af996e
+  data.tar.gz: c87f65e73d1112165856ca4cfd4aea432521a1c3
+SHA512:
+  metadata.gz: 63f31c1154cef0c30b595840939018ca3dc505acb924fd09be384fd78d99303815642803ab461187e1ede1d906281564a20cff478c4cbdc5573cc2c477e544f4
+  data.tar.gz: 5b504063d8583fcf30c112a8fbc9445c027120f9d09ec721d466df10e0b28c4401d59f48ed93a93c3753c36cb73606b7801b6db4f0d20711558319fb5bcf9c02

data/LICENSE

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2017, Matthias Geier
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/lib/sideload.rb

@@ -0,0 +1,51 @@
+require "redis"
+require "sideload/validation_error"
+require "sideload/config"
+require "sideload/redis"
+require "sideload/path"
+require "sideload/github"
+
+module Sideload
+  extend self
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def init
+    c = Config.new
+    yield(c)
+    return c
+  end
+
+  def update!(sources)
+    scope, arg, _config, validate = sources.shift
+    mod = const_get(scope.to_s.capitalize)
+    contents = mod.read(*arg)
+    unless sources.empty?
+      next_layer = update!(sources)
+      unless next_layer.nil?
+        (contents.keys | next_layer.keys).each do |key|
+          if !next_layer.has_key?(key)
+            mod.with(arg, key) do |fp, t|
+              mod.delete(fp, t)
+            end
+          else
+            mod.with(arg, key) do |fp, t|
+              mod.write(fp, t, next_layer[key], &validate)
+            end
+          end
+        end
+        contents = mod.read(arg)
+      end
+    end
+    contents.each(&Proc.new) if block_given?
+    return contents
+  rescue ValidationError => e
+    if @logger
+      @logger.error { [e.class, e.message].inspect }
+      @logger.debug { e.backtrace.join("\n") }
+    end
+    return nil
+  end
+end

data/lib/sideload/config.rb

@@ -0,0 +1,41 @@
+module Sideload
+  class Config
+    ALLOWED = %i[path github redis]
+
+    attr_reader :sources, :packer, :unpacker
+
+    def initialize
+      @sources = []
+      @packer = ->(f) { f }
+      @unpacker = ->(f) { f }
+    end
+
+    def pack
+      @packer = Proc.new
+    end
+
+    def unpack
+      @unpacker = Proc.new
+    end
+
+    def validate
+      @validate = Proc.new
+    end
+
+    def source(scope, arg, **config)
+      if !ALLOWED.include?(scope)
+        raise "scope #{scope.inspect} not in #{ALLOWED.inspect}"
+      end
+      @sources << [
+        scope,
+        arg,
+        config,
+        (block_given? ? Proc.new : nil) || @validate
+      ]
+    end
+
+    def update!
+      Sideload.update!(@sources.dup, &(block_given? ? Proc.new : nil))
+    end
+  end
+end

data/lib/sideload/github.rb

@@ -0,0 +1,91 @@
+require "net/http"
+require "base64"
+require "json"
+
+module Sideload
+  module Github
+    extend self
+
+    GITHUB_V3 = URI("https://api.github.com")
+
+    def credentials=(arr)
+      @user, @pass = arr
+    end
+
+    def read(repo, path)
+      sha = navigate_to(repo, path)
+      return traverse(repo, sha)
+    end
+
+    def with(path, fname)
+      raise RuntimeError.new("not implemented")
+    end
+
+    def write(full_path, target, content)
+      raise RuntimeError.new("not implemented")
+    end
+
+    def delete(full_path, target)
+      raise RuntimeError.new("not implemented")
+    end
+
+    private
+
+    def cred!
+      raise RuntimeError.new("no github credentials set") if @user.nil?
+    end
+
+    def traverse(repo, sha, path = [])
+      return get_tree(repo, sha).reduce({}) do |acc, node|
+        name = node["path"]
+        case node["type"]
+        when "blob"
+          acc[(path + [name]).join("/")] = get_blob(node["url"])
+        when "tree"
+          acc.merge!(traverse(repo, node["sha"], path + [name]))
+        end
+        next acc
+      end
+    end
+
+    def navigate_to(repo, path)
+      return path.split("/").reduce("master") do |acc, folder|
+        next acc if folder.empty?
+        get_tree(repo, acc)&.detect { |e| e["type"] && e["path"] == folder }&.
+          []("sha")
+      end
+    end
+
+    def get_blob(url)
+      cred!
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request.basic_auth(@user, @pass)
+      response = http.request(request)
+      if response.is_a?(Net::HTTPOK)
+        return Base64.decode64(JSON.parse(response.body)&.[]("content"))
+      else
+        puts response, url
+        return nil
+      end
+    end
+
+    def get_tree(repo, folder)
+      cred!
+      uri = URI.join(GITHUB_V3, "/repos/#{repo}/git/trees/#{folder}")
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request.basic_auth(@user, @pass)
+      response = http.request(request)
+      if response.is_a?(Net::HTTPOK)
+        return JSON.parse(response.body)&.[]("tree")
+      else
+        puts response, uri
+        return nil
+      end
+    end
+  end
+end

data/lib/sideload/path.rb

@@ -0,0 +1,39 @@
+module Sideload
+  module Path
+    extend self
+
+    def read(path)
+      dir = File.expand_path(path)
+      return Dir[dir + "/**/*.*"].map do |fname|
+        [fname.sub(dir + "/", ""), File.open(fname) { |f| f.read }]
+      end.to_h
+    end
+
+    def with(path, fname)
+      *dirs, target = fname.split("/")
+      full_path = dirs.reduce(File.expand_path(path)) do |acc, dir|
+        f = File.join(acc, dir)
+        Dir.mkdir(f) unless File.directory?(f)
+        next f
+      end
+      yield(full_path, target)
+      dirs.reverse.each do |dir|
+        break if File.basename(full_path) != dir
+        break unless Dir[full_path + "/*.*"].empty?
+        Dir.rmdir(full_path)
+        full_path = File.dirname(full_path)
+      end
+    end
+
+    def write(full_path, target, content)
+      if block_given? && !yield(content)
+        raise ValidationError.new(self, "#{full_path}/#{target}", content)
+      end
+      File.open(File.join(full_path, target), "w") { |f| f.print content }
+    end
+
+    def delete(full_path, target)
+      File.delete(File.join(full_path, target))
+    end
+  end
+end

data/lib/sideload/redis.rb

@@ -0,0 +1,34 @@
+module Sideload
+  module Redis
+    extend self
+
+    def db!(**config)
+      @redis = ::Redis.new(**config)
+    end
+
+    def db
+      @redis || db!
+    end
+
+    def read(path)
+      return db.keys(path + "*").map do |key|
+        [key.sub(path, ""), db.get(key)]
+      end.to_h
+    end
+
+    def with(path, fname)
+      yield(path, fname)
+    end
+
+    def write(full_path, target, content)
+      if block_given? && !yield(content)
+        raise ValidationError.new(self, "#{full_path}/#{target}", content)
+      end
+      db.set(File.join(full_path, target), content)
+    end
+
+    def delete(full_path, target)
+      db.del(File.join(full_path, target))
+    end
+  end
+end

data/lib/sideload/validation_error.rb

@@ -0,0 +1,7 @@
+module Sideload
+  class ValidationError < RuntimeError
+    def initialize(klass, path, content)
+      super("validation error in #{klass}: #{path}\n#{content}")
+    end
+  end
+end

data/lib/sideload/version.rb

@@ -0,0 +1,3 @@
+module Sideload
+  VERSION = "0.1.0"
+end

data/test/sideload/config_spec.rb

@@ -0,0 +1,107 @@
+describe "Sideload::Config" do
+  before do
+    @config = Sideload::Config.new
+  end
+
+  after do
+    @config = nil
+  end
+
+  describe ".new" do
+    it "initializes with empty sources" do
+      assert_equal [], @config.sources
+    end
+
+    it "initializes with forwarding packer and unpacker" do
+      assert_equal 25, @config.packer.call(25)
+      assert_equal 25, @config.unpacker.call(25)
+    end
+  end
+
+  describe "#pack" do
+    it "overrides the packer with proc" do
+      blk = ->(f) { f * 2 }
+      assert_equal 25, @config.packer.call(25)
+      @config.pack(&blk)
+      assert_equal 50, @config.packer.call(25)
+    end
+  end
+
+  describe "#unpack" do
+    it "overrides the unpacker with proc" do
+      blk = ->(f) { f * 2 }
+      assert_equal 25, @config.unpacker.call(25)
+      @config.unpack(&blk)
+      assert_equal 50, @config.unpacker.call(25)
+    end
+  end
+
+  describe "#source" do
+    it "allows path, web and redis as scopes" do
+      %i[path github redis].each do |scope|
+        raised = false
+        begin
+          @config.source(scope, *["arg"])
+        rescue
+          raised = true
+        end
+        refute raised
+      end
+    end
+
+    it "fails on any other scope" do
+      raised = false
+      begin
+        @config.source(:nuu, "arg")
+      rescue
+        raised = true
+      end
+      assert raised
+    end
+
+    it "stores at the back of sources" do
+      @config.source(:path, "first/")
+      assert_equal [[:path, "first/", {}, nil]], @config.sources
+      @config.source(:path, "second/")
+      assert_equal [[:path, "first/", {}, nil], [:path, "second/", {}, nil]],
+        @config.sources
+    end
+
+    it "stores any config opts" do
+      @config.source(:path, "first/", any: "opt", with: 1)
+      assert_equal [[:path, "first/", {any: "opt", with: 1}, nil]],
+        @config.sources
+    end
+
+    it "uses a generic validate when available, otherwise given block" do
+      @config.validate { |f| f == 25 }
+      @config.source(:path, nil)
+      validator = @config.sources.last.last
+      assert validator.call(25)
+      refute validator.call(26)
+      @config.source(:path, nil) { |f| f == 26 }
+      validator = @config.sources.last.last
+      assert validator.call(26)
+      refute validator.call(25)
+    end
+  end
+
+  describe "#update!" do
+    it "forwards a block and the sources array to sideload" do
+      @config.source(:path, nil)
+      ran = 0
+      test_val = Proc.new do |sources|
+        assert_equal @config.sources, sources
+        ran += 1
+      end
+      Sideload.stub(:update!, test_val, 25) do
+        @config.update!
+        @config.update! do |f|
+          assert_equal 25, f
+          ran += 1
+        end
+      end
+      assert_equal 3, ran
+    end
+  end
+end

data/test/sideload/github_spec.rb

@@ -0,0 +1,33 @@
+describe "Sideload::Github" do
+  after do
+    Sideload::Github.instance_variable_set("@user", nil)
+    Sideload::Github.instance_variable_set("@pass", nil)
+  end
+
+  describe ".credentials=" do
+    it "sets credentials for github" do
+      Sideload::Github.credentials = ["moo", "bar"]
+      assert_equal "moo", Sideload::Github.instance_variable_get("@user")
+      assert_equal "bar", Sideload::Github.instance_variable_get("@pass")
+    end
+  end
+
+  describe ".read" do
+    it "raises when no credentials are set" do
+      raised = false
+      begin
+        Sideload::Github.read("matthias-geier/sideload", "sample_sources")
+      rescue RuntimeError => e
+        assert_equal "no github credentials set", e.message
+        raised = true
+      end
+      assert raised
+    end
+
+    it "lists keys and contents under a path" do
+      Sideload::Github.credentials = ["e608c053ca8068b09a5fbc7f337cb22f11cf7725", "x-oauth-basic"]
+      assert_equal({"toot.json" => "[]\n"}, Sideload::Github.
+        read("matthias-geier/sideload", "sample_sources"))
+    end
+  end
+end

data/test/sideload/path_spec.rb

@@ -0,0 +1,62 @@
+describe "Sideload::Path" do
+  describe ".read" do
+    it "reads files on a given path" do
+      assert_equal({"toot.json" => "[]\n"},
+        Sideload::Path.read("sample_sources/"))
+    end
+  end
+
+  describe ".with" do
+    it "creates and clears missing folders for the target" do
+      refute File.exist?("sample_sources/cookie")
+      Sideload::Path.with("sample_sources/", "cookie/joe.json") do |_path, file|
+        assert File.exist?("sample_sources/cookie")
+        assert_equal "joe.json", file
+      end
+      refute File.exist?("sample_sources/cookie")
+    end
+  end
+
+  describe ".write" do
+    it "validates against content with block" do
+      m = Module.new { def self.print(_); end }
+      File.stub(:open, nil, m) do
+        raised = false
+        ran = 0
+        begin
+          Sideload::Path.write("", "", "{}") do |c|
+            assert_equal "{}", c
+            ran += 1
+            true
+          end
+          Sideload::Path.write("", "", "{}") do |c|
+            assert_equal "{}", c
+            ran += 1
+            false
+          end
+        rescue Sideload::ValidationError
+          raised = true
+        end
+        assert raised
+        assert_equal 2, ran
+      end
+    end
+
+    it "writes content to a file" do
+      m = Module.new { def self.print(c); @ran = 1; @content = c; end }
+      File.stub(:open, nil, m) do
+        Sideload::Path.write("", "", "{}")
+      end
+      assert_equal 1, m.instance_variable_get("@ran")
+      assert_equal "{}", m.instance_variable_get("@content")
+    end
+  end
+
+  describe ".delete" do
+    it "deletes a file" do
+      File.stub(:delete, ->(path) { assert_equal "s/m.json", path }) do
+        Sideload::Path.delete("s", "m.json")
+      end
+    end
+  end
+end

data/test/sideload/redis_spec.rb

@@ -0,0 +1,84 @@
+describe "Sideload::Path" do
+  after do
+    db = Sideload::Redis.db
+    db.del("moo/bar", "blu/bar", "s/m.json", "x/m.json")
+    Sideload::Redis.remove_instance_variable("@redis")
+  end
+
+  describe ".db" do
+    it "initiaizes and delivers the same redis instance" do
+      assert Sideload::Redis.db.is_a?(Redis)
+      assert_equal Sideload::Redis.db, Sideload::Redis.db
+    end
+  end
+
+  describe ".db!" do
+    it "initializes a new redis instance with params" do
+      db = Sideload::Redis.db
+      Sideload::Redis.db!(db: 3)
+      assert Sideload::Redis.db.is_a?(Redis)
+      refute_equal db, Sideload::Redis.db
+    end
+  end
+
+  describe ".read" do
+    it "lists keys and contents under a path" do
+      db = Sideload::Redis.db
+      db.set("moo/bar", "{}")
+      db.set("blu/bar", "{}")
+      assert_equal({"bar" => "{}"}, Sideload::Redis.read("moo/"))
+    end
+  end
+
+  describe ".with" do
+    it "yields with given params" do
+      ran = false
+      Sideload::Redis.with(1, 2) do |path, fname|
+        ran = true
+        assert_equal 1, path
+        assert_equal 2, fname
+      end
+      assert ran
+    end
+  end
+
+  describe ".write" do
+    it "validates against content with block" do
+      raised = false
+      ran = 0
+      begin
+        Sideload::Redis.write("s", "m.json", "{}") do |c|
+          assert_equal "{}", c
+          ran += 1
+          true
+        end
+        Sideload::Redis.write("x", "m.json", "{}") do |c|
+          assert_equal "{}", c
+          ran += 1
+          false
+        end
+      rescue Sideload::ValidationError
+        raised = true
+      end
+      db = Sideload::Redis.db
+      assert_equal "{}", db.get("s/m.json")
+      refute db.exists("x/m.json")
+      assert raised
+      assert_equal 2, ran
+    end
+
+    it "writes content to key" do
+      Sideload::Redis.write("s", "m.json", "{}")
+      assert_equal "{}", Sideload::Redis.db.get("s/m.json")
+    end
+  end
+
+  describe ".delete" do
+    it "deletes a key" do
+      db = Sideload::Redis.db
+      db.set("s/m.json", "{}")
+      Sideload::Redis.delete("s", "m.json")
+      refute db.exists("s/m.json")
+    end
+  end
+end

data/test/test_runner.rb

@@ -0,0 +1,7 @@
+require "sideload"
+require "json"
+require "minitest/autorun"
+
+Dir["test/**/*_spec.rb"].each do |f|
+  load f
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: sideload
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matthias Geier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+description: Offers a safe way to copy (config) files through differentstorages
+email: 'mayutamano@gmail.com '
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/sideload.rb
+- lib/sideload/config.rb
+- lib/sideload/github.rb
+- lib/sideload/path.rb
+- lib/sideload/redis.rb
+- lib/sideload/validation_error.rb
+- lib/sideload/version.rb
+- test/sideload/config_spec.rb
+- test/sideload/github_spec.rb
+- test/sideload/path_spec.rb
+- test/sideload/redis_spec.rb
+- test/test_runner.rb
+homepage: https://github.com/matthias-geier/sideload
+licenses:
+- BSD-2-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Validated data copying
+test_files:
+- test/sideload/redis_spec.rb
+- test/sideload/path_spec.rb
+- test/sideload/github_spec.rb
+- test/sideload/config_spec.rb
+- test/test_runner.rb