pgai 0.2.4 → 1.0.0.alpha2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ class ExternalCommandManager
5
+ def initialize(env_names, command, logger: nil)
6
+ @env_names = env_names
7
+ @command = command
8
+ @logger = logger || Pgai::Commander.instance.logger
9
+ end
10
+
11
+ def run
12
+ Signal.trap("INT", "IGNORE")
13
+
14
+ prepare(env_names)
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :env_names, :command, :logger
20
+
21
+ def prepare(envs, variables = {})
22
+ if (env_name = envs.pop)
23
+ Pgai::Commander.instance.with_env(env_name) do |env|
24
+ Pgai::CloneManager.new(env).prepare do |cached_clone|
25
+ variables["#{env_name.to_s.upcase}_DATABASE_URL"] = cached_clone.database_url
26
+ prepare(envs, variables)
27
+ end
28
+ end
29
+ else
30
+ execute(variables)
31
+ end
32
+ end
33
+
34
+ def execute(variables)
35
+ pid = fork do
36
+ exec(variables, *command)
37
+ end
38
+ Process.wait(pid)
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Pgai
6
+ module Port
7
+ class Allocator
8
+ START_PORT = 5000
9
+ FINISH_PORT = 9000
10
+ LOCALHOST = "127.0.0.1"
11
+
12
+ def self.pick
13
+ new.pick
14
+ end
15
+
16
+ def initialize(start: START_PORT, finish: FINISH_PORT)
17
+ @start = start
18
+ @finish = finish
19
+ end
20
+
21
+ def pick
22
+ (start + jitter).upto(finish) do |port|
23
+ return port if available?(port)
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ attr_reader :start, :finish
30
+
31
+ def jitter
32
+ rand((finish - start) / 2).floor
33
+ end
34
+
35
+ def available?(port)
36
+ TCPServer.new(LOCALHOST, port).close
37
+ true
38
+ rescue Errno::EADDRINUSE
39
+ false
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "net/ssh"
5
+
6
+ module Pgai
7
+ module Port
8
+ class Forwarder
9
+ LOCALHOST = "127.0.0.1"
10
+
11
+ attr_reader :local_port, :remote_host, :remote_port, :logger
12
+
13
+ def initialize(local_port, remote_host, remote_port, logger: Pgai::Commander.instance.logger)
14
+ @local_port = local_port
15
+ @remote_host = remote_host
16
+ @remote_port = remote_port
17
+ @logger = logger
18
+ @child, @parent = Socket.pair(:UNIX, :DGRAM)
19
+ end
20
+
21
+ def start
22
+ return if ready?
23
+
24
+ debug "starting"
25
+ start_ssh_connection
26
+ wait_until_ready
27
+ debug "ready to accept connections"
28
+ end
29
+
30
+ def stop
31
+ return unless @pid
32
+
33
+ @parent.write("exit")
34
+ debug "shutting down"
35
+ Process.wait(@pid)
36
+ debug "exit"
37
+ @pid = nil
38
+ end
39
+
40
+ private
41
+
42
+ def start_ssh_connection
43
+ @pid = Process.fork do
44
+ Signal.trap("INT", "IGNORE")
45
+ @parent.close
46
+
47
+ Net::SSH.start(remote_host) do |ssh|
48
+ ssh.forward.local(local_port, LOCALHOST, remote_port)
49
+ ssh.loop(0.1) { keep_running? }
50
+ end
51
+ end
52
+ end
53
+
54
+ def keep_running?
55
+ @child.read_nonblock(10).empty?.tap do |value|
56
+ debug "shutdown command received"
57
+ end
58
+ rescue
59
+ true
60
+ end
61
+
62
+ def ready?
63
+ !!TCPSocket.new(LOCALHOST, local_port)
64
+ rescue Errno::ECONNREFUSED
65
+ false
66
+ end
67
+
68
+ def wait_until_ready
69
+ until ready?
70
+ sleep 0.02
71
+ end
72
+ end
73
+
74
+ def debug(message)
75
+ logger.debug "#{logger_inspect} #{message}"
76
+ end
77
+
78
+ def logger_inspect
79
+ "[#{self.class}(#{local_port}:#{remote_host}:#{remote_port})]:"
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent-ruby"
4
+
5
+ module Pgai
6
+ module Port
7
+ class Manager
8
+ def initialize(logger: Pgai::Commander.instance.logger)
9
+ @logger = logger
10
+ @executor = Concurrent::IndirectImmediateExecutor.new
11
+ end
12
+
13
+ def forward(local_port, host, port)
14
+ forwarder = start(local_port, host, port)
15
+ yield
16
+ ensure
17
+ forwarder&.stop
18
+ end
19
+
20
+ def start(local_port, host, port)
21
+ fw = Port::Forwarder.new(local_port, host, port, logger: @logger)
22
+ @executor.post(fw, &:start)
23
+ fw
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ class PsqlManager
5
+ def initialize(cached_clone, logger:)
6
+ @cached_clone = cached_clone
7
+ @logger = logger
8
+ end
9
+
10
+ def run
11
+ Signal.trap("INT", "IGNORE")
12
+ logger.info "Data state at: #{cached_clone.data_state_at}"
13
+
14
+ psql
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :cached_clone, :logger
20
+
21
+ def psql
22
+ psql_pid = fork do
23
+ exec("psql #{cached_clone.connection_string}")
24
+ end
25
+
26
+ start_caffeinate(psql_pid)
27
+ Process.wait(psql_pid)
28
+ end
29
+
30
+ def start_caffeinate(pid)
31
+ return if `which caffeinate`.to_s.empty?
32
+
33
+ caffeinate_pid = Process.spawn("caffeinate -is -w #{pid}")
34
+ Process.detach(caffeinate_pid)
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+
5
+ module Pgai
6
+ module Resources
7
+ module Attributes
8
+ def self.included(base)
9
+ base.extend ClassMethods
10
+ attr_reader :attributes
11
+ end
12
+
13
+ class Attribute
14
+ FALSE_VALUES = [
15
+ "0", "f", "F", "false", "FALSE",
16
+ "off", "OFF", "NO", "no"
17
+ ].to_set.freeze
18
+
19
+ TYPES = {
20
+ string: ->(value) { value.to_s },
21
+ integer: ->(value) { value.to_i },
22
+ decimal: ->(value) { value.to_f },
23
+ boolean: ->(value) { !FALSE_VALUES.include?(value.to_s) },
24
+ datetime: ->(value) { ::Time.at(::Time.new(value.to_s, in: "UTC")) }
25
+ }
26
+
27
+ attr_reader :name
28
+ attr_reader :type
29
+ attr_reader :default
30
+
31
+ def initialize(name, type, default)
32
+ @name = name
33
+ @type = type
34
+ @default = default
35
+
36
+ yield self if block_given?
37
+ end
38
+
39
+ def cast(value)
40
+ TYPES.fetch(type).call(value)
41
+ end
42
+
43
+ def cast_or_default(user_value)
44
+ value = default
45
+ value = cast(user_value) unless user_value.nil?
46
+ value = value.call if value.respond_to?(:call)
47
+ value
48
+ end
49
+ end
50
+
51
+ module ClassMethods
52
+ def inherited(subclass)
53
+ attributes.each do |attr|
54
+ subclass.attribute attr.name, attr.type, default: attr.default
55
+ end
56
+ end
57
+
58
+ def attributes
59
+ @attributes ||= []
60
+ end
61
+
62
+ def attribute(name, type, default: nil)
63
+ Attribute.new(name, type, default) do |attribute|
64
+ attributes << attribute
65
+ generate_attribute_methods(attribute)
66
+ end
67
+ end
68
+
69
+ def generate_attribute_methods(attribute)
70
+ define_method(attribute.name) do
71
+ instance_variable_get(:"@#{attribute.name}")
72
+ end
73
+
74
+ define_method("#{attribute.name}=") do |value|
75
+ instance_variable_set(:"@#{attribute.name}", value)
76
+ attributes[attribute.name.to_sym] = value
77
+ end
78
+ end
79
+ end
80
+
81
+ def initialize(user_attributes = {})
82
+ @attributes = {}
83
+
84
+ self.class.attributes.each do |attribute|
85
+ assign_attribute(attribute, user_attributes[attribute.name])
86
+ end
87
+ end
88
+
89
+ def assign_attribute(attribute, user_value)
90
+ value = attribute.cast_or_default(user_value)
91
+
92
+ public_send(:"#{attribute.name}=", value)
93
+ end
94
+
95
+ def assign_attributes(user_attributes)
96
+ user_attributes.each do |key, value|
97
+ attribute = self.class.attributes.find { |attr| attr.name == key }
98
+ raise "shit" unless attribute
99
+
100
+ assign_attribute(attribute, value)
101
+ end
102
+ end
103
+
104
+ def has_attribute?(name)
105
+ respond_to?(name) && respond_to?(:"#{name}=")
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+
5
+ module Pgai
6
+ module Resources
7
+ module Local
8
+ class BaseRecord
9
+ include Attributes
10
+ extend Forwardable
11
+
12
+ def_delegators :klass, :backend, :record_type
13
+
14
+ class << self
15
+ def all
16
+ backend.all(record_type).map do |attributes|
17
+ new(attributes)
18
+ end
19
+ end
20
+
21
+ def find(id)
22
+ attributes = backend.find(record_type, id)
23
+ return unless attributes
24
+
25
+ record = new(attributes)
26
+ yield(record) if block_given?
27
+ record
28
+ end
29
+
30
+ def delete(id)
31
+ backend.delete(record_type, id)
32
+ true
33
+ end
34
+
35
+ def record_type
36
+ :"#{name.split("::").last.downcase}s"
37
+ end
38
+
39
+ def backend
40
+ Pgai::Commander.instance.store
41
+ end
42
+ end
43
+
44
+ def save
45
+ backend.save(record_type, attributes, key: store_key)
46
+ true
47
+ end
48
+
49
+ def delete
50
+ klass.delete(id)
51
+ end
52
+
53
+ def klass
54
+ self.class
55
+ end
56
+
57
+ private
58
+
59
+ def store_key
60
+ :id
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Clone < BaseRecord
7
+ attribute :id, :string
8
+ attribute :host, :string
9
+ attribute :remote_host, :string
10
+ attribute :port, :integer
11
+ attribute :password, :string
12
+ attribute :user, :string
13
+ attribute :dbname, :string
14
+ attribute :created_at, :datetime
15
+ attribute :data_state_at, :datetime
16
+
17
+ def connection_string
18
+ "'host=#{host} port=#{local_port} user=#{user} dbname=#{dbname} password=#{password}'"
19
+ end
20
+
21
+ def database_url
22
+ "postgresql://#{user}:#{password}@#{host}:#{local_port}/#{dbname}"
23
+ end
24
+
25
+ def with_port_forward(manager = Pgai::Commander.instance.port_manager)
26
+ manager.forward(local_port, remote_host, port) do
27
+ yield self
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def local_port
34
+ @local_port ||= Port::Allocator.pick
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Configuration < BaseRecord
7
+ attribute :id, :string, default: "config"
8
+ attribute :clone_prefix, :string
9
+ attribute :access_token, :string
10
+
11
+ def self.default
12
+ find("config")
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Environment < BaseRecord
7
+ attribute :id, :string
8
+ attribute :alias, :string
9
+ attribute :port, :integer
10
+ attribute :dbname, :string
11
+ attribute :access_token, :string
12
+ attribute :clone_prefix, :string
13
+
14
+ def client
15
+ @client ||= Pgai::Client.new(
16
+ token: access_token,
17
+ host: "http://127.0.0.1:#{local_port}"
18
+ )
19
+ end
20
+
21
+ def prepare
22
+ with_port_forward do
23
+ Resources::Remote::BaseRecord.with_client(client) do
24
+ yield(self)
25
+ end
26
+ end
27
+ end
28
+
29
+ def with_port_forward(manager = Pgai::Commander.instance.port_manager)
30
+ manager.forward(local_port, id, port) do
31
+ yield self
32
+ end
33
+ end
34
+
35
+ def expected_cloning_time
36
+ @expected_cloning_time ||= client.expected_cloning_time
37
+ end
38
+
39
+ def local_port
40
+ @local_port ||= Port::Allocator.pick
41
+ end
42
+
43
+ def clone_id
44
+ @clone_id ||= "#{clone_prefix}_#{self.alias}"
45
+ end
46
+
47
+ private
48
+
49
+ def store_key
50
+ :alias
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class BaseRecord
7
+ include Attributes
8
+
9
+ class << self
10
+ def client
11
+ @@client ||= nil
12
+ end
13
+
14
+ def client=(value)
15
+ @@client = value
16
+ end
17
+
18
+ def with_client(new_client)
19
+ previous = client
20
+ self.client = new_client
21
+ yield
22
+ ensure
23
+ self.client = previous
24
+ end
25
+ end
26
+
27
+ def client
28
+ self.class.client
29
+ end
30
+
31
+ def refresh_attributes(data)
32
+ assign_attributes(data)
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class Clone < BaseRecord
7
+ attribute :id, :string
8
+ attribute :protected, :boolean, default: false
9
+ attribute :created_at, :datetime
10
+ attribute :delete_at, :datetime
11
+
12
+ attr_accessor :db_object
13
+ attr_accessor :snapshot
14
+ attr_accessor :status
15
+ attr_accessor :metadata
16
+
17
+ class << self
18
+ def find(id)
19
+ data = client.get(File.join(path, id))
20
+ raise Pgai::ResourceNotFound unless data.key?(:id)
21
+
22
+ new(data.slice(:id, :protected)) do |resource|
23
+ resource.refresh_attributes(data)
24
+ end
25
+ end
26
+
27
+ def path
28
+ "clone"
29
+ end
30
+ end
31
+
32
+ def save
33
+ body = {
34
+ id: id,
35
+ snapshot: {
36
+ id: snapshot.id
37
+ },
38
+ protected: protected,
39
+ db: {
40
+ username: db_object.username,
41
+ password: db_object.password,
42
+ restricted: db_object.restricted
43
+ }
44
+ }
45
+
46
+ data = client.post(self.class.path, body)
47
+ refresh_attributes(data)
48
+
49
+ self
50
+ end
51
+
52
+ def ready?
53
+ status.code == "OK"
54
+ end
55
+
56
+ def refresh
57
+ data = client.get(object_path)
58
+ raise Pgai::ResourceNotFound unless data.key?(:id)
59
+
60
+ refresh_attributes(data)
61
+
62
+ self
63
+ end
64
+
65
+ def delete
66
+ data = client.delete(object_path)
67
+ refresh_attributes(data) unless data.empty?
68
+
69
+ self
70
+ end
71
+
72
+ def reset
73
+ client.post(object_path("reset"))
74
+
75
+ self
76
+ end
77
+
78
+ def refresh_attributes(data)
79
+ assign_attributes(data.slice(:delete_at, :created_at))
80
+ ensure_metadata.refresh_attributes(data.fetch(:metadata))
81
+ ensure_status.refresh_attributes(data.fetch(:status))
82
+ ensure_db_object.refresh_attributes(data.fetch(:db))
83
+ ensure_snapshot.refresh_attributes(data.fetch(:snapshot))
84
+ end
85
+
86
+ def status_message
87
+ status&.message
88
+ end
89
+
90
+ private
91
+
92
+ def object_path(*fragments)
93
+ File.join(self.class.path, id, *fragments)
94
+ end
95
+
96
+ def ensure_metadata
97
+ metadata || self.metadata = CloneMetadata.new
98
+ end
99
+
100
+ def ensure_status
101
+ status || self.status = CloneStatus.new
102
+ end
103
+
104
+ def ensure_db_object
105
+ db_object || self.db_object = DbObject.new
106
+ end
107
+
108
+ def ensure_snapshot
109
+ snapshot || self.snapshot = Snapshot.new
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class CloneMetadata < BaseRecord
7
+ attribute :clone_diff_size, :integer
8
+ attribute :logical_size, :integer
9
+ attribute :cloning_time, :decimal
10
+ attribute :max_idle_minutes, :integer
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class CloneStatus < BaseRecord
7
+ attribute :code, :string
8
+ attribute :message, :string
9
+ end
10
+ end
11
+ end
12
+ end