pgai 0.2.4 → 1.0.0.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ class ExternalCommandManager
5
+ def initialize(env_names, command, logger: nil)
6
+ @env_names = env_names
7
+ @command = command
8
+ @logger = logger || Pgai::Commander.instance.logger
9
+ end
10
+
11
+ def run
12
+ Signal.trap("INT", "IGNORE")
13
+
14
+ prepare(env_names)
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :env_names, :command, :logger
20
+
21
+ def prepare(envs, variables = {})
22
+ if (env_name = envs.pop)
23
+ Pgai::Commander.instance.with_env(env_name) do |env|
24
+ Pgai::CloneManager.new(env).prepare do |cached_clone|
25
+ variables["#{env_name.to_s.upcase}_DATABASE_URL"] = cached_clone.database_url
26
+ prepare(envs, variables)
27
+ end
28
+ end
29
+ else
30
+ execute(variables)
31
+ end
32
+ end
33
+
34
+ def execute(variables)
35
+ pid = fork do
36
+ exec(variables, *command)
37
+ end
38
+ Process.wait(pid)
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Pgai
6
+ module Port
7
+ class Allocator
8
+ START_PORT = 5000
9
+ FINISH_PORT = 9000
10
+ LOCALHOST = "127.0.0.1"
11
+
12
+ def self.pick
13
+ new.pick
14
+ end
15
+
16
+ def initialize(start: START_PORT, finish: FINISH_PORT)
17
+ @start = start
18
+ @finish = finish
19
+ end
20
+
21
+ def pick
22
+ (start + jitter).upto(finish) do |port|
23
+ return port if available?(port)
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ attr_reader :start, :finish
30
+
31
+ def jitter
32
+ rand((finish - start) / 2).floor
33
+ end
34
+
35
+ def available?(port)
36
+ TCPServer.new(LOCALHOST, port).close
37
+ true
38
+ rescue Errno::EADDRINUSE
39
+ false
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "net/ssh"
5
+
6
+ module Pgai
7
+ module Port
8
+ class Forwarder
9
+ LOCALHOST = "127.0.0.1"
10
+
11
+ attr_reader :local_port, :remote_host, :remote_port, :logger
12
+
13
+ def initialize(local_port, remote_host, remote_port, logger: Pgai::Commander.instance.logger)
14
+ @local_port = local_port
15
+ @remote_host = remote_host
16
+ @remote_port = remote_port
17
+ @logger = logger
18
+ @child, @parent = Socket.pair(:UNIX, :DGRAM)
19
+ end
20
+
21
+ def start
22
+ return if ready?
23
+
24
+ debug "starting"
25
+ start_ssh_connection
26
+ wait_until_ready
27
+ debug "ready to accept connections"
28
+ end
29
+
30
+ def stop
31
+ return unless @pid
32
+
33
+ @parent.write("exit")
34
+ debug "shutting down"
35
+ Process.wait(@pid)
36
+ debug "exit"
37
+ @pid = nil
38
+ end
39
+
40
+ private
41
+
42
+ def start_ssh_connection
43
+ @pid = Process.fork do
44
+ Signal.trap("INT", "IGNORE")
45
+ @parent.close
46
+
47
+ Net::SSH.start(remote_host) do |ssh|
48
+ ssh.forward.local(local_port, LOCALHOST, remote_port)
49
+ ssh.loop(0.1) { keep_running? }
50
+ end
51
+ end
52
+ end
53
+
54
+ def keep_running?
55
+ @child.read_nonblock(10).empty?.tap do |value|
56
+ debug "shutdown command received"
57
+ end
58
+ rescue
59
+ true
60
+ end
61
+
62
+ def ready?
63
+ !!TCPSocket.new(LOCALHOST, local_port)
64
+ rescue Errno::ECONNREFUSED
65
+ false
66
+ end
67
+
68
+ def wait_until_ready
69
+ until ready?
70
+ sleep 0.02
71
+ end
72
+ end
73
+
74
+ def debug(message)
75
+ logger.debug "#{logger_inspect} #{message}"
76
+ end
77
+
78
+ def logger_inspect
79
+ "[#{self.class}(#{local_port}:#{remote_host}:#{remote_port})]:"
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "concurrent-ruby"
4
+
5
+ module Pgai
6
+ module Port
7
+ class Manager
8
+ def initialize(logger: Pgai::Commander.instance.logger)
9
+ @logger = logger
10
+ @executor = Concurrent::IndirectImmediateExecutor.new
11
+ end
12
+
13
+ def forward(local_port, host, port)
14
+ forwarder = start(local_port, host, port)
15
+ yield
16
+ ensure
17
+ forwarder&.stop
18
+ end
19
+
20
+ def start(local_port, host, port)
21
+ fw = Port::Forwarder.new(local_port, host, port, logger: @logger)
22
+ @executor.post(fw, &:start)
23
+ fw
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ class PsqlManager
5
+ def initialize(cached_clone, logger:)
6
+ @cached_clone = cached_clone
7
+ @logger = logger
8
+ end
9
+
10
+ def run
11
+ Signal.trap("INT", "IGNORE")
12
+ logger.info "Data state at: #{cached_clone.data_state_at}"
13
+
14
+ psql
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :cached_clone, :logger
20
+
21
+ def psql
22
+ psql_pid = fork do
23
+ exec("psql #{cached_clone.connection_string}")
24
+ end
25
+
26
+ start_caffeinate(psql_pid)
27
+ Process.wait(psql_pid)
28
+ end
29
+
30
+ def start_caffeinate(pid)
31
+ return if `which caffeinate`.to_s.empty?
32
+
33
+ caffeinate_pid = Process.spawn("caffeinate -is -w #{pid}")
34
+ Process.detach(caffeinate_pid)
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+
5
+ module Pgai
6
+ module Resources
7
+ module Attributes
8
+ def self.included(base)
9
+ base.extend ClassMethods
10
+ attr_reader :attributes
11
+ end
12
+
13
+ class Attribute
14
+ FALSE_VALUES = [
15
+ "0", "f", "F", "false", "FALSE",
16
+ "off", "OFF", "NO", "no"
17
+ ].to_set.freeze
18
+
19
+ TYPES = {
20
+ string: ->(value) { value.to_s },
21
+ integer: ->(value) { value.to_i },
22
+ decimal: ->(value) { value.to_f },
23
+ boolean: ->(value) { !FALSE_VALUES.include?(value.to_s) },
24
+ datetime: ->(value) { ::Time.at(::Time.new(value.to_s, in: "UTC")) }
25
+ }
26
+
27
+ attr_reader :name
28
+ attr_reader :type
29
+ attr_reader :default
30
+
31
+ def initialize(name, type, default)
32
+ @name = name
33
+ @type = type
34
+ @default = default
35
+
36
+ yield self if block_given?
37
+ end
38
+
39
+ def cast(value)
40
+ TYPES.fetch(type).call(value)
41
+ end
42
+
43
+ def cast_or_default(user_value)
44
+ value = default
45
+ value = cast(user_value) unless user_value.nil?
46
+ value = value.call if value.respond_to?(:call)
47
+ value
48
+ end
49
+ end
50
+
51
+ module ClassMethods
52
+ def inherited(subclass)
53
+ attributes.each do |attr|
54
+ subclass.attribute attr.name, attr.type, default: attr.default
55
+ end
56
+ end
57
+
58
+ def attributes
59
+ @attributes ||= []
60
+ end
61
+
62
+ def attribute(name, type, default: nil)
63
+ Attribute.new(name, type, default) do |attribute|
64
+ attributes << attribute
65
+ generate_attribute_methods(attribute)
66
+ end
67
+ end
68
+
69
+ def generate_attribute_methods(attribute)
70
+ define_method(attribute.name) do
71
+ instance_variable_get(:"@#{attribute.name}")
72
+ end
73
+
74
+ define_method("#{attribute.name}=") do |value|
75
+ instance_variable_set(:"@#{attribute.name}", value)
76
+ attributes[attribute.name.to_sym] = value
77
+ end
78
+ end
79
+ end
80
+
81
+ def initialize(user_attributes = {})
82
+ @attributes = {}
83
+
84
+ self.class.attributes.each do |attribute|
85
+ assign_attribute(attribute, user_attributes[attribute.name])
86
+ end
87
+ end
88
+
89
+ def assign_attribute(attribute, user_value)
90
+ value = attribute.cast_or_default(user_value)
91
+
92
+ public_send(:"#{attribute.name}=", value)
93
+ end
94
+
95
+ def assign_attributes(user_attributes)
96
+ user_attributes.each do |key, value|
97
+ attribute = self.class.attributes.find { |attr| attr.name == key }
98
+ raise "shit" unless attribute
99
+
100
+ assign_attribute(attribute, value)
101
+ end
102
+ end
103
+
104
+ def has_attribute?(name)
105
+ respond_to?(name) && respond_to?(:"#{name}=")
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+
5
+ module Pgai
6
+ module Resources
7
+ module Local
8
+ class BaseRecord
9
+ include Attributes
10
+ extend Forwardable
11
+
12
+ def_delegators :klass, :backend, :record_type
13
+
14
+ class << self
15
+ def all
16
+ backend.all(record_type).map do |attributes|
17
+ new(attributes)
18
+ end
19
+ end
20
+
21
+ def find(id)
22
+ attributes = backend.find(record_type, id)
23
+ return unless attributes
24
+
25
+ record = new(attributes)
26
+ yield(record) if block_given?
27
+ record
28
+ end
29
+
30
+ def delete(id)
31
+ backend.delete(record_type, id)
32
+ true
33
+ end
34
+
35
+ def record_type
36
+ :"#{name.split("::").last.downcase}s"
37
+ end
38
+
39
+ def backend
40
+ Pgai::Commander.instance.store
41
+ end
42
+ end
43
+
44
+ def save
45
+ backend.save(record_type, attributes, key: store_key)
46
+ true
47
+ end
48
+
49
+ def delete
50
+ klass.delete(id)
51
+ end
52
+
53
+ def klass
54
+ self.class
55
+ end
56
+
57
+ private
58
+
59
+ def store_key
60
+ :id
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Clone < BaseRecord
7
+ attribute :id, :string
8
+ attribute :host, :string
9
+ attribute :remote_host, :string
10
+ attribute :port, :integer
11
+ attribute :password, :string
12
+ attribute :user, :string
13
+ attribute :dbname, :string
14
+ attribute :created_at, :datetime
15
+ attribute :data_state_at, :datetime
16
+
17
+ def connection_string
18
+ "'host=#{host} port=#{local_port} user=#{user} dbname=#{dbname} password=#{password}'"
19
+ end
20
+
21
+ def database_url
22
+ "postgresql://#{user}:#{password}@#{host}:#{local_port}/#{dbname}"
23
+ end
24
+
25
+ def with_port_forward(manager = Pgai::Commander.instance.port_manager)
26
+ manager.forward(local_port, remote_host, port) do
27
+ yield self
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def local_port
34
+ @local_port ||= Port::Allocator.pick
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Configuration < BaseRecord
7
+ attribute :id, :string, default: "config"
8
+ attribute :clone_prefix, :string
9
+ attribute :access_token, :string
10
+
11
+ def self.default
12
+ find("config")
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Local
6
+ class Environment < BaseRecord
7
+ attribute :id, :string
8
+ attribute :alias, :string
9
+ attribute :port, :integer
10
+ attribute :dbname, :string
11
+ attribute :access_token, :string
12
+ attribute :clone_prefix, :string
13
+
14
+ def client
15
+ @client ||= Pgai::Client.new(
16
+ token: access_token,
17
+ host: "http://127.0.0.1:#{local_port}"
18
+ )
19
+ end
20
+
21
+ def prepare
22
+ with_port_forward do
23
+ Resources::Remote::BaseRecord.with_client(client) do
24
+ yield(self)
25
+ end
26
+ end
27
+ end
28
+
29
+ def with_port_forward(manager = Pgai::Commander.instance.port_manager)
30
+ manager.forward(local_port, id, port) do
31
+ yield self
32
+ end
33
+ end
34
+
35
+ def expected_cloning_time
36
+ @expected_cloning_time ||= client.expected_cloning_time
37
+ end
38
+
39
+ def local_port
40
+ @local_port ||= Port::Allocator.pick
41
+ end
42
+
43
+ def clone_id
44
+ @clone_id ||= "#{clone_prefix}_#{self.alias}"
45
+ end
46
+
47
+ private
48
+
49
+ def store_key
50
+ :alias
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class BaseRecord
7
+ include Attributes
8
+
9
+ class << self
10
+ def client
11
+ @@client ||= nil
12
+ end
13
+
14
+ def client=(value)
15
+ @@client = value
16
+ end
17
+
18
+ def with_client(new_client)
19
+ previous = client
20
+ self.client = new_client
21
+ yield
22
+ ensure
23
+ self.client = previous
24
+ end
25
+ end
26
+
27
+ def client
28
+ self.class.client
29
+ end
30
+
31
+ def refresh_attributes(data)
32
+ assign_attributes(data)
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class Clone < BaseRecord
7
+ attribute :id, :string
8
+ attribute :protected, :boolean, default: false
9
+ attribute :created_at, :datetime
10
+ attribute :delete_at, :datetime
11
+
12
+ attr_accessor :db_object
13
+ attr_accessor :snapshot
14
+ attr_accessor :status
15
+ attr_accessor :metadata
16
+
17
+ class << self
18
+ def find(id)
19
+ data = client.get(File.join(path, id))
20
+ raise Pgai::ResourceNotFound unless data.key?(:id)
21
+
22
+ new(data.slice(:id, :protected)) do |resource|
23
+ resource.refresh_attributes(data)
24
+ end
25
+ end
26
+
27
+ def path
28
+ "clone"
29
+ end
30
+ end
31
+
32
+ def save
33
+ body = {
34
+ id: id,
35
+ snapshot: {
36
+ id: snapshot.id
37
+ },
38
+ protected: protected,
39
+ db: {
40
+ username: db_object.username,
41
+ password: db_object.password,
42
+ restricted: db_object.restricted
43
+ }
44
+ }
45
+
46
+ data = client.post(self.class.path, body)
47
+ refresh_attributes(data)
48
+
49
+ self
50
+ end
51
+
52
+ def ready?
53
+ status.code == "OK"
54
+ end
55
+
56
+ def refresh
57
+ data = client.get(object_path)
58
+ raise Pgai::ResourceNotFound unless data.key?(:id)
59
+
60
+ refresh_attributes(data)
61
+
62
+ self
63
+ end
64
+
65
+ def delete
66
+ data = client.delete(object_path)
67
+ refresh_attributes(data) unless data.empty?
68
+
69
+ self
70
+ end
71
+
72
+ def reset
73
+ client.post(object_path("reset"))
74
+
75
+ self
76
+ end
77
+
78
+ def refresh_attributes(data)
79
+ assign_attributes(data.slice(:delete_at, :created_at))
80
+ ensure_metadata.refresh_attributes(data.fetch(:metadata))
81
+ ensure_status.refresh_attributes(data.fetch(:status))
82
+ ensure_db_object.refresh_attributes(data.fetch(:db))
83
+ ensure_snapshot.refresh_attributes(data.fetch(:snapshot))
84
+ end
85
+
86
+ def status_message
87
+ status&.message
88
+ end
89
+
90
+ private
91
+
92
+ def object_path(*fragments)
93
+ File.join(self.class.path, id, *fragments)
94
+ end
95
+
96
+ def ensure_metadata
97
+ metadata || self.metadata = CloneMetadata.new
98
+ end
99
+
100
+ def ensure_status
101
+ status || self.status = CloneStatus.new
102
+ end
103
+
104
+ def ensure_db_object
105
+ db_object || self.db_object = DbObject.new
106
+ end
107
+
108
+ def ensure_snapshot
109
+ snapshot || self.snapshot = Snapshot.new
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class CloneMetadata < BaseRecord
7
+ attribute :clone_diff_size, :integer
8
+ attribute :logical_size, :integer
9
+ attribute :cloning_time, :decimal
10
+ attribute :max_idle_minutes, :integer
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgai
4
+ module Resources
5
+ module Remote
6
+ class CloneStatus < BaseRecord
7
+ attribute :code, :string
8
+ attribute :message, :string
9
+ end
10
+ end
11
+ end
12
+ end