qwe 0.0.0

data/lib/qwe/db/record.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Qwe::DB
+  class Record
+    include DRb::DRbUndumped
+
+    def _dump(...)
+      ""
+    end
+
+    def self._load(...)
+      nil
+    end
+
+    def detach_at
+      @detach_at || keep
+    end
+
+    def keep
+      @detach_at = Time.now + Worker.instance.detach_timeout
+    end
+
+    def should_detach?
+      detach_at < Time.now
+    end
+
+    def object=(obj)
+      raise "Object for record #{id} is already set" if object
+      @object = obj
+    end
+
+    def id=(id)
+      raise "Object for record #{id} is already set" if self.id
+      @id = id
+    end
+
+    def commit(str)
+      @meta["commits_length"] += str.count("\n")
+      commits_file.write(str)
+      unless str[-1] == "\n"
+        commits_file.write("\n")
+        @meta["commits_length"] += 1
+      end
+      self
+    rescue => e
+      log "Can't write commit #{str}: #{e.full_message}"
+      self
+    end
+
+    def commits_length=(v)
+      @meta["commits_length"] = v
+    end
+
+    def commits_length
+      @meta["commits_length"]
+    end
+
+    def meta_file
+      File.join(@dir, "meta")
+    end
+
+    attr_reader :dir, :object, :id, :meta
+
+    def commits_file
+      @commits_file ||= CommitsFile.new(@dir)
+    end
+
+    def dump
+      Marshal.dump(object)
+    end
+
+    def dump_object
+      File.binwrite(File.join(dir, commits_length.to_s), dump)
+    end
+
+    def create_object(klass)
+      self.object = klass.new
+      object.init if klass.include?(Qwe::Mixins::Root)
+      dump_object
+    end
+
+    def initialize(id, server_dir, create: nil, **args)
+      @dir = File.join(server_dir, "records", id.to_s)
+      self.id = id
+
+      if create
+        Qwe::DB::Server.mkdir(dir)
+        @meta = args
+        @meta["commits_length"] = 0
+        File.write(meta_file, JSON.generate(args))
+        if create.is_a?(Class)
+          create_object(create)
+        elsif create.is_a?(Symbol)
+          create_object(Object.module_eval(create.to_s))
+        elsif create.is_a?(String)
+          self.object = Marshal.load(create)
+          File.binwrite(File.join(dir, "0"), create)
+        else
+          self.object = create
+          dump_object
+        end
+      else
+        @meta = JSON.parse(File.read(meta_file))
+        self.object = Marshal.load(File.binread(File.join(dir, "dump")))
+      end
+
+      if object.respond_to?(:record=)
+        object.record = self
+      end
+    end
+
+    def save
+      save!
+    rescue => e
+      log "Can't save record #{id}: #{e.full_message} #{e.backtrace}"
+    end
+
+    def save!
+      commits_file.close
+      File.binwrite(File.join(dir, "dump"), dump)
+      File.write(meta_file, JSON.generate(@meta))
+    end
+
+    def commits(commit_id = nil)
+      commits_file.read(commit_id)
+    end
+
+    def object_at(commit_id)
+      o = Marshal.load File.read(File.join(dir, "0"))
+      o.instance_eval commits(commit_id)
+      o
+    end
+
+    def detach
+      Worker.instance.detach(id, self)
+    end
+
+    def fork(commit_id = nil)
+      log "Fork at commit id #{commit_id}"
+      commit_id ||= commits_length
+
+      zero = File.binread(File.join(dir, "0"))
+      rec = Worker.instance.allocate_record(zero)
+
+      commits = commits(commit_id)
+      rec.commits_length = commit_id + 1
+
+      begin
+        rec.object.instance_eval(commits)
+      rescue ScriptError, StandardError => e
+        log e.full_message
+        log "Commits are:\n\n#{commits}\n\n"
+      end
+
+      if rec.commits.length > 0
+        log "Commits should not produce another commits:", "\n" + rec.commits
+        raise "Commits loop detected"
+      end
+
+      rec.commits_file.write(commits)
+      rec
+    end
+
+    def obj_eval(rb)
+      object.instance_eval(rb)
+    end
+
+    def archive
+      archive!
+    rescue => e
+      log "Error archiving #{id}: #{e.full_message}"
+    end
+
+    def archive!
+      log "Archive record #{id}"
+      commits_file.archive!
+    end
+  end
+end

data/lib/qwe/db/server.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Qwe::DB
+  class Server
+    include DRb::DRbUndumped
+    include Qwe::Mixins::Process
+
+    DEFAULT_PORT = 3228
+    DEFAULT_HOST = "druby://localhost"
+
+    attr_reader :uri, :dir, :host, :port, :records_length, :threads_count
+
+    def self.default_uri
+      "#{DEFAULT_HOST}:#{DEFAULT_PORT}"
+    end
+
+    def initialize(
+      dir: "#{Dir.pwd}/qwe_db",
+      host: DEFAULT_HOST,
+      port: DEFAULT_PORT,
+      require_file: nil,
+      no_jit: false,
+      threads: Etc.nprocessors,
+      gc_interval: 0,
+      detach_timeout: 300
+    )
+      @port = port
+      @host = host
+      @dir = dir
+      @uri = "#{@host}:#{@port}"
+      @threads_count = threads
+
+      @worker_args = [RbConfig.ruby]
+      @worker_args.push "--jit" unless no_jit
+      @worker_args += ["#{__dir__}/../../spawn_worker.rb", uri, dir, gc_interval.to_s, detach_timeout.to_s]
+      @worker_args.push require_file if require_file
+
+      @workers = {}
+      @records = {}
+
+      log "Initialized server in #{dir}"
+    end
+
+    def start
+      self.class.mkdir(dir)
+      d = File.join(dir, "records")
+      self.class.mkdir(d)
+      @records_length = (Dir.new(d).children.map(&:to_i).max || 0) + 1
+      @drb = DRb.start_service(uri, self)
+
+      trap_stop_signals
+
+      log "Started server at #{@drb.uri}, records_length = #{records_length}, pid = #{Process.pid}"
+
+      @threads_count.times do
+        spawn_worker(wait: false)
+      end
+
+      @ready = true
+
+      @drb.thread.join
+    end
+
+    def ready?
+      @ready || false
+    end
+
+    def [](id)
+      id = id.to_i
+      w = @records[id]
+      if w
+        if w.is_a?(Thread)
+          w.value
+        else
+          w[id]
+        end
+      else
+        load(id)
+      end
+    end
+
+    def workers_length
+      @workers.length
+    end
+
+    def self.mkdir(d)
+      Dir.mkdir(d) unless Dir.exist?(d)
+    end
+
+    def records_length=(len)
+      @records_length = len
+      File.write(rl_file, len)
+    end
+
+    def stop
+      return if @stopping
+      @stopping = true
+      log "Stopping"
+      @workers.each do |pid, worker|
+        log "Terminate worker #{pid}"
+        Process.kill("TERM", pid)
+      rescue Errno::ESRCH
+        log "Worker #{pid} is already dead"
+      rescue => e
+        log "Error terminating worker #{pid}: #{e.full_message}"
+      end
+      exit
+    end
+
+    SPAWN_WORKER_TIMEOUT = 50
+
+    def spawn_worker(wait: true)
+      pid = spawn(*@worker_args)
+      Process.detach(pid)
+
+      @workers[pid] = Thread.new do
+        sleep SPAWN_WORKER_TIMEOUT
+        raise "Worker did not start after #{SPAWN_WORKER_TIMEOUT}s timeout"
+      end
+      @workers[pid].join if wait
+      pid
+    end
+
+    def pick_worker(attempt = 0)
+      pid = @workers.keys.sample
+      if @workers[pid].is_a? Thread
+        @workers[pid].join
+      end
+      @workers[pid]
+    rescue => e
+      log "Error picking worker: #{e.full_message}, attempt #{attempt}"
+      raise "Give up" if attempt > 2
+      del_worker(pid)
+      spawn_worker
+      pick_worker(attempt + 1)
+    end
+
+    def set_worker_uri(uri, pid)
+      t = @workers[pid]
+      @workers[pid] = DRbObject.new_with_uri(uri)
+      log "Worker #{pid} is up"
+      t.exit
+    end
+
+    def crash_worker(pid, message)
+      @workers[pid].raise message
+      @workers.delete(pid)
+    end
+
+    def del_worker(pid)
+      if @workers[pid].is_a?(Thread)
+        @workers[pid].exit
+      end
+      @workers.delete(pid)
+    end
+
+    def create(klass)
+      id = records_length
+      w = pick_worker
+      w.create(id, klass)
+      log "Create record #{id} in worker #{w.uri}"
+      @records[id] = w
+      @records_length += 1
+      id
+    end
+
+    def detach(id)
+      @records[id].detach(id)
+      nil
+    end
+
+    def detach_record(id)
+      @records.delete(id)
+    end
+
+    def load(id)
+      @records[id] = Thread.new do
+        w = pick_worker
+        log "Load record #{id} in worker #{w.uri}"
+        obj = w.load(id)
+        @records[id] = w
+        obj
+      end
+      @records[id].value
+    end
+
+    def allocate_record(pid)
+      id = records_length
+      @records_length += 1
+      @records[id] = @workers[pid]
+      log "Allocate record #{id}"
+      id
+    end
+
+    def destroy(id)
+      destroy!(id)
+    rescue => e
+      log "Can't destroy - #{e.full_message}"
+      nil
+    end
+
+    def destroy!(id)
+      throw "Record #{id} is in use" if @records[id]
+      FileUtils.rm_r(File.join(dir, "records", id.to_s))
+      log "Destroyed #{id}"
+      nil
+    end
+  end
+end

data/lib/qwe/db/worker.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Qwe::DB
+  class Worker
+    include DRb::DRbUndumped
+    include Qwe::Mixins::Process
+
+    attr_reader :server_dir, :server, :records, :gc_interval, :detach_timeout, :requirements
+
+    POLL_INTERVAL = 20
+
+    def initialize(server_url, server_dir, gc_interval = 0, detach_timeout = 300, requirements = nil)
+      @server_dir = server_dir
+      @server = DRbObject.new_with_uri(server_url)
+      @gc_interval = gc_interval.to_i
+      @detach_timeout = detach_timeout.to_i
+
+      @requirements = requirements
+      begin
+        require requirements if requirements
+      rescue ScriptError, StandardError => e
+        server.crash_worker(Process.pid, "Error requiring '#{requirements}': #{e.full_message}")
+        exit
+      end
+
+      Qwe::Function.compile_all
+
+      @records = {}
+      @object_refs = {}
+    end
+
+    def start_poll
+      @poll_thread = Thread.new do
+        loop do
+          sleep POLL_INTERVAL
+          @records.keys.each do |id|
+            r = @records[id]
+            detach(id, r) if r.should_detach?
+          end
+        end
+      end
+    end
+
+    def detach(id, record = nil)
+      record ||= @records[id]
+      @server.detach_record(id)
+      @object_refs.delete(id)
+      @records.delete(id)
+      record.save
+      log "Detach record #{id} from worker #{uri}"
+      nil
+    end
+
+    def start_gc
+      @gc_thread = Thread.new do
+        loop do
+          sleep @gc_interval
+          next if @records.empty?
+          GC.start
+        end
+      end
+    end
+
+    def uri
+      @drb&.uri
+    end
+
+    def work
+      @drb = DRb.start_service("druby://localhost:0", self)
+      log "Started worker at #{@drb.uri}, pid = #{Process.pid}, requirements = #{@requirements}"
+      @server.set_worker_uri(uri, Process.pid)
+
+      start_poll
+      start_gc if @gc_interval > 0
+
+      trap_stop_signals
+
+      @drb.thread.join
+    end
+
+    def create(id, klass)
+      @records[id] = Qwe::DB::Record.new(id, server_dir, create: klass)
+    end
+
+    def load(id)
+      @records[id] = Qwe::DB::Record.new(id, server_dir)
+      DRbObject.new(@records[id].object)
+    end
+
+    def [](i)
+      @object_refs[i] ||= DRbObject.new(@records[i].object)
+    end
+
+    def stop
+      return if @stopping
+      @stopping = true
+      log "Stopping"
+      @records.keys.each { |id| @records[id].save }
+      exit
+    end
+
+    def allocate_record(zero)
+      log "Allocate record for #{zero.class}"
+      id = server.allocate_record(Process.pid)
+      records[id] = Qwe::DB::Record.new(id, server_dir, create: zero)
+    end
+
+    def self.instance
+      @@instance
+    end
+
+    def self.spawn(*)
+      @@instance = Qwe::DB::Worker.new(*)
+      @@instance.work
+    end
+
+    def e(str)
+      instance_eval str
+    end
+  end
+end

data/lib/qwe/db.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "db/commits_file"
+require_relative "db/server"
+require_relative "db/worker"
+require_relative "db/record"
+
+module Qwe
+  module DB
+    # Server #
+
+    def self.serve(*a1, **a2)
+      @@srv = Server.new(*a1, **a2)
+      @@srv.start
+    end
+
+    def self.stop
+      @@srv.stop
+    end
+
+    # Client #
+
+    def self.connect(uri = Server.default_uri)
+      @@server = DRbObject.new_with_uri(uri)
+      300.times do
+        if server.ready?
+          return server
+        end
+        sleep 0.1
+      rescue
+        sleep 0.1
+      end
+      raise "Couldn't connect to server in more than 30s"
+    end
+
+    def self.server
+      @@server ||= DRbObject.new_with_uri(Server.default_uri)
+    end
+
+    def self.[](id)
+      server[id]
+    end
+
+    def self.create(...)
+      server.create(...)
+    end
+
+    def self.destroy(...)
+      server.destroy(...)
+    end
+
+    def self.destroy!(...)
+      server.destroy!(...)
+    end
+
+    def self.detach(id)
+      server.detach(id)
+      nil
+    end
+  end
+end

data/lib/qwe/function.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Qwe
+  class Function
+    attr_accessor :klass
+    attr_reader :name
+
+    def name=(n)
+      @name = n.to_sym
+    end
+
+    def self.instances
+      @@instances ||= []
+    end
+
+    def self.compile_all
+      instances.each(&:compile)
+    end
+
+    def initialize(klass, name)
+      self.klass = klass
+      self.name = name
+      self.class.instances.push(self)
+      Qwe[klass, :functions, name] = self
+
+      unless klass.respond_to?(:compile)
+        klass.define_singleton_method(:compile) do
+          Qwe[self, :functions].each do |n, f|
+            f.compile
+          end
+        end
+      end
+    end
+
+    def args
+      @args ||= {}
+    end
+
+    def arg(name, default = nil)
+      args[name] = default
+    end
+
+    def keywords
+      @keywords ||= {}
+    end
+
+    def keyword(name, default = nil)
+      keywords[name] = default
+    end
+
+    def stages
+      @stages ||= []
+    end
+
+    def stage(code)
+      @compiled = false
+      stages.push(code) if code
+    end
+
+    def code
+      rb = "def #{name}(" + \
+        ((args.to_a.map { |a| "#{a[0]} = #{a[1].to_rb}" }) + \
+        (keywords.to_a.map { |a| "#{a[0]}: #{a[1].to_rb}" })).join(", ") + \
+        ")\n"
+      stages.each do |s|
+        rb += "  " + s + "\n"
+      end
+      rb += "end"
+    end
+
+    def compile
+      return if @compiled
+
+      begin
+        klass.module_eval(code)
+      rescue ScriptError, StandardError => e
+        log "Error compiling #{klass}.#{name}, code is: \n#{code}\n"
+        raise e
+      end
+
+      @compiled = true
+    end
+
+    def compiled?
+      @compiled
+    end
+
+    def inspect
+      "\nFunction #{klass}.#{name}\n#{code}\n"
+    end
+
+    def self.compile(klass, name, &block)
+      f = Qwe::Function.new(klass, name)
+      yield f
+      f.compile
+    end
+  end
+end

data/lib/qwe/mixins/process.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Qwe::Mixins::Process
+  def trap_stop_signals
+    Signal.trap("INT") do
+      log "SIGINT, stop"
+      stop
+    end
+    Signal.trap("TERM") do
+      log "SIGTERM, stop"
+      stop
+    end
+  end
+end