collab 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae9c092826099d0d81012ba20446f93518d9f998136d110ae71141bed2a58e22
-  data.tar.gz: e6de0fca8c103f77bdf6677fb0ab993a8740eae2baf7292f28cf601a5946f4b7
+  metadata.gz: f8c6f4ca46b51f7fd05bacdea192bbd5ec7979b6c44f7528665beda336fb6d43
+  data.tar.gz: e2a0748fe7e1686a36a7ace934153433b1a499c18f8a0c930ed689564f9eb90e
 SHA512:
-  metadata.gz: a9db9beb2c46f7b4818fd494075ddb31ee68bcb137c3a347f71acf754efc3788afbaf2b67248b9e7b1a9bd80a780015a2a1db5a5778533823d32a83134550954
-  data.tar.gz: d65efb8027b52bf44c14e3a36829f66a845130dca3cffaf002b0ec0fe8cdeed2a137fd6e671aff6555c52ed03705b66e72f54fab5b2872aaa30d872f4f83d476
+  metadata.gz: 36feda3ab2009c1a954434893a1ca89eed1f9492b08a20af7872d720abceaf4351398b16a9c6ba63cccf76f318ba387b3be29c85cf0616aba52a20f8fed7f71c
+  data.tar.gz: bd79ab6de7c2c24185f581031d4b25f673d5ef037f2f3794d4b06cb2b2906c772227d8ad9aa3ec4b63b28d82f0a48f2fdd3a54f6835db919b67b628b0c83fdb0

data/app/jobs/collab/commit_job.rb

@@ -0,0 +1,7 @@
+class Collab::CommitJob < ::Collab.config.base_job.constantize
+  queue_as :default
+
+  def perform(document, data)
+    document.commits.from_json(data).apply!
+  end
+end

data/lib/collab.rb

@@ -1,19 +1,15 @@
-require "collab/railtie"
+require "collab/version"
+require "collab/config"
+require "collab/js"
+require "collab/engine"
 
 module Collab
-  def self.config
-    @config ||= Collab::Config.new
-    return @config unless block_given?
-    yield @config
-  end
-
-  autoload "Config", "collab/config"
-  autoload "Bridge", "collab/bridge"
+  autoload "Channel", "collab/channel"
   autoload "HasCollaborativeDocument", "collab/has_collaborative_document"
   
   module Models
     autoload "Base", "collab/models/base"
     autoload "Document", "collab/models/document"
-    autoload "DocumentTransaction", "collab/models/document_transaction"
+    autoload "Commit", "collab/models/commit"
   end
 end

data/lib/collab/channel.rb

@@ -10,19 +10,20 @@ module Collab
 
       stream_for document
       
-      transactions = document.transactions
-                              .where("document_version > ?", starting_version)
-                              .order(document_version: :asc)
-                              .load
+      commits = document.commits
+                        .where("document_version > ?", starting_version)
+                        .order(document_version: :asc)
+                        .load
               
-      raise "invalid version" unless transactions.first.document_version == (starting_version + 1) unless transactions.empty?
-
-      transactions.lazy.map(&:as_json).each(method(:transmit))
+      unless commits.empty?
+        raise "invalid version" unless commits.first.document_version == (starting_version + 1)
+        commits.lazy.map(&:as_json).each(method(:transmit))
+      end
     end
 
-    def submit(data)
-      authorize_submit!(data)
-      document.perform_transaction_later(data)
+    def commit(data)
+      authorize_commit!(data)
+      document.commit_later(data)
     end
 
     def unsubscribed

data/lib/collab/config.rb

@@ -1,5 +1,34 @@
 module Collab
+  @config_mutex = Mutex.new
+
+  def self.config
+    if block_given?
+      @config_mutex.synchronize do
+        @config ||= ::Collab::Config.new
+        raise "[Collab] Tried to configure gem after first use" if @config.frozen?
+        yield @config
+      end
+    else
+      raise "[Collab] Missing configuration - Have you run `rails g collab:install` yet?" unless @config
+      @config.freeze # really weird stuff could happen if the config changes after first use, so freeze config
+    end
+  end
+
   class Config
-    attr_accessor :max_transactions, :application_job, :queue_document_transaction_job_as, :schema_package, :application_record, :document_transaction_model, :document_model, :channel_name
+    attr_accessor :base_record,
+                  :base_job,
+                  :channel,
+                  :commit_job,
+                  :commit_model,
+                  :document_model,
+                  :max_commit_history_length,
+                  :num_js_processes,
+                  :schema_package
+
+    def initialize
+      self.commit_job = "Collab::CommitJob"
+      self.document_model = "Collab::Models::Document"
+      self.commit_model = "Collab::Models::Commit"
+    end
   end
 end

data/lib/collab/engine.rb

@@ -0,0 +1,6 @@
+if defined?(Rails)
+  module Collab
+      class Engine < ::Rails::Engine
+      end
+  end
+end

data/lib/collab/has_collaborative_document.rb

@@ -8,7 +8,7 @@ module Collab
 
         define_method attach_as do
           super() || begin
-            document = self.__send__("build_#{attach_as}", schema_name: schema, document: blank_document.dup)
+            document = self.__send__("build_#{attach_as}", schema_name: schema, content: blank_document.dup)
             document.save!
             document
           end

data/lib/collab/js.rb

@@ -0,0 +1,93 @@
+require "json"
+
+module Collab
+  module JS
+    @queue = Queue.new
+    @queue_initialized = false
+    @queue_initialization_mutex = Mutex.new
+
+    class <<self
+      def queue
+        initialize_queue unless @queue_initialized
+        @queue
+      end
+
+      # Calls the block given with a JS process acquired from the queue
+      # Will block until a JS process is available
+      def with_js
+        js = queue.pop
+        yield js
+      ensure
+        queue << js
+      end
+
+      def call(name, data = nil, schema_name:)
+        with_js { |js| js.call(name, *arguments, &block) }
+      end
+      
+      def apply_commit(document, commit, schema_name:)
+        call("applyCommit", {doc: document, commit: commit}, schema_name: schema_name)
+      end
+
+      def html_to_document(html, schema_name:)
+        call("htmlToDoc", html, schema_name: schema_name)
+      end
+
+      def document_to_html(document, schema_name:)
+        call("docToHtml", document, schema_name: schema_name)
+      end
+
+      private 
+      # Thread-safe initialization of the NodeJS process queue
+      def initialize_queue
+        @queue_initialization_mutex.synchronize do
+          unless @queue_initialized
+            ::Collab.config.num_js_processes.times { @queue << ::Collab::JS::JSProcess.new }
+            @queue_initialized = true
+          end
+        end
+      end
+    end
+
+    class JSProcess
+      def initialize
+         @node = if defined?(Rails)
+                   Dir.chdir(Rails.root) { open_node }
+                 else
+                   open_node
+                 end
+      end
+
+      def call(name, data = nil, schema_name:)
+        req = {name: name, data: data, schemaPackage: ::Collab.config.schema_package, schemaName: schema_name}
+        @node.puts(JSON.generate(req))
+        res = JSON.parse(@node.gets)
+        raise ::Collab::JS::JSRuntimeError.new(res["error"]) if res["error"]
+        res["result"]
+      end
+
+      private
+      def open_node
+        IO.popen(["node", "-e", "require('rails-collab-server')"], "r+")
+      end
+    end
+
+    class JSRuntimeError < StandardError
+      def initialize(data)
+        @js_backtrace = data["stack"].split("\n").map{|f| "JavaScript #{f.strip}"} if data["stack"]
+
+        super(data["name"] + ": " + data["message"])
+      end
+
+      def backtrace
+        return unless  val = super
+        
+        if @js_backtrace
+          @js_backtrace + val
+        else
+          val
+        end
+      end
+    end
+  end
+end

data/lib/collab/models/base.rb

@@ -1,5 +1,5 @@
 module Collab
-  class Models::Base < ::Collab.config.application_record.constantize
+  class Models::Base < ::Collab.config.base_record.constantize
     self.abstract_class = true
     self.table_name_prefix = 'collab_'
   end

data/lib/collab/models/commit.rb

@@ -0,0 +1,53 @@
+module Collab
+  class Models::Commit < ::Collab::Models::Base
+    belongs_to :document, class_name: ::Collab.config.document_model
+
+    validates :steps, presence: true
+    validates :document_version, presence: true
+    validates :ref, length: { maximum: 36 }
+
+    after_create_commit :broadcast
+
+    def self.from_json(data)
+      new(document_version: data["v"]&.to_i, steps: data["steps"], ref: data["ref"])
+    end
+
+    def as_json
+      {
+        v: document_version,
+        steps: steps,
+        ref: ref
+      }
+    end
+
+    def apply_later
+      raise "cannot apply persisted commit" if self.persisted?
+      raise "commit not valid" unless self.valid?
+      return false if self.document.document_version != self.document_version
+
+      ::Collab.config.commit_job.constantize.perform_later(self.document, as_json)
+    end
+
+    def apply!
+      raise "cannot apply persisted commit" if self.persisted?
+      raise "commit not valid" unless self.valid?
+      return false if self.document.document_version != self.document_version # optimization, prevents need to apply lock if outdated
+
+      self.document.with_lock do
+        return false if self.document.document_version != self.document_version
+        
+        return false unless result = ::Collab::JS.apply_commit(self.document, self.to_json, schema_name: self.document.schema_name)
+        
+        self.document.document = result["doc"]
+        self.document.document_version = self.document_version
+        
+        self.document.save!
+        self.save!
+      end
+    end
+
+    def broadcast
+      ::Collab.config.channel_name.constantize.broadcast_to(document, as_json)
+    end
+  end
+end

data/lib/collab/models/document.rb

@@ -1,73 +1,52 @@
 module Collab
   class Models::Document < ::Collab::Models::Base
     belongs_to :attached, polymorphic: true
-    has_many :transactions, class_name: ::Collab.config.document_transaction_model, foreign_key: :document_id
+    has_many :commits, class_name: ::Collab.config.commit_model, foreign_key: :document_id
 
-    validates :document, presence: true
+    validates :content, presence: true
     validates :document_version, presence: true, numericality: {only_integer: true, greater_than_or_equal_to: 0}
     validates :schema_name, presence: true
 
-    before_save :nullify_serialized_html, unless: :serialized_html_fresh?
-    after_save :delete_old_transactions
+    after_save :delete_old_commits
 
-    # The already-serialized html version of this document
-    def serialized_html
-      return super if serialized_html_fresh?
-    end
-
-    def serialized_html_fresh?
-      serialized_html_version == document_version
-    end
-
-    # Serialize the document to html - will be cached if possible (somewhat expensive)
+    # Serialize the document to html, uses cached if possible.
+    # Note that this may lock the document
     def to_html
       return serialized_html if serialized_html
 
-      serialized_html = ::Collab::Bridge.current.document_to_html(self.document, schema_name: schema_name)
-      self.update! serialized_html: serialized_html, serialized_html_version: document_version
-      serialized_html
+      serialized_version = self.document_version
+      ::Collab::JS.document_to_html(self.content, schema_name: schema_name).tap do |serialized_html|
+        Thread.new do # use a thread to prevent deadlocks and avoid incuring the cost of an inline-write
+          self.with_lock do
+            self.update_attribute(:serialized_html, serialized_html) if serialized_version == self.version and self.serialized_html.nil?
+          end
+        end
+      end
     end
 
     def from_html(html)
-      self.document = ::Collab::Bridge.current.html_to_document(html, schema_name: schema_name)
-    end
-
-    def perform_transaction_later(data)
-      ::Collab::DocumentTransactionJob.perform_later(@document, data)
+      self.content = ::Collab::JS.html_to_document(html, schema_name: schema_name)
     end
 
-    def apply_transaction_now(data)
-      return unless data["v"].is_a?(Integer)
-
-      return if (data["v"] - 1) != self.document_version # if expired between queue and perform, there is no need to aquire a lock. this is an optimization - not a guarantee
-      with_lock do
-        return if (data["v"] - 1) != self.document_version # now that we've aquired a lock to the record, ensure that we're still accessing the correct version
-        
-        transaction_result = ::Collab::Bridge.current.apply_transaction(self.document, data, schema_name: self.schema_name)
-        return unless transaction_result # check to make sure the transaction succeeded
-        
-        self.document = transaction_result["doc"]
-        self.document_version = data["v"]
-        save!
-
-        transactions.create! document_version: self.document_version, steps: data["steps"], ref: data["ref"]
-      end
+    def commit_later(data)
+      commits.from_json(data).apply_later
     end
 
     def as_json
-      {id: id, document: document, version: self.document_version}
+      {id: id, content: content, version: document_version}
     end
 
-    private
-
-    def nullify_serialized_html
+    def content_will_change!
+      super
       self.serialized_html = nil
     end
 
-    def delete_old_transactions
-      cutoff = document_version - ::Collab.config.max_transactions
+    private
+
+    def delete_old_commits
+      cutoff = document_version - ::Collab.config.max_commit_history_length
       return if cutoff <= 0
-      transactions.where("document_version < ?", cutoff).delete_all
+      commits.where("document_version < ?", cutoff).delete_all
     end
   end
 end

data/lib/collab/version.rb

@@ -1,3 +1,3 @@
 module Collab
-  VERSION = '0.2.0'
+  VERSION = '0.3.1'
 end

data/lib/generators/collab/install/install_generator.rb

@@ -17,8 +17,10 @@ module Collab
     end
 
     def copy_files
+      @primary_key_type = Rails.application.config.generators.active_record[:primary_key_type]
+
       migration_template(
-        "create_collab_tables.rb",
+        "create_collab_tables.rb.erb",
         "db/migrate/create_collab_tables.rb",
       )
       

data/lib/generators/collab/install/templates/channel.rb

@@ -4,16 +4,19 @@ class CollabDocumentChannel < ApplicationCable::Channel
   private
 
   # Find the document to subscribe to based on the params passed to the channel
-  # Authorization may also be performed here (raise an error)
+  # Authorization may also be performed here (raise an error to prevent subscription)
   def find_document
     Collab::Models::Document.find(params[:document_id]).tap do |document|
-      raise "authorization failed"
+      # TODO: Replace with your own authorization logic
+      raise "authorization not implemented"
     end
   end
 
-  # Called when a client submits a transaction in order to update a document
-  # You should throw an error if unauthorized
-  def authorize_submit!
-    raise "authorization failed"
+  # Called a commit is first received for processing
+  # Throw an error to prevent the commit from being processed
+  # You should consider adding some type of rate-limiting here
+  def authorize_commit!(data)
+    # TODO: Replace with your own authorization logic
+    raise "authorization not implemented"
   end
 end

data/lib/generators/collab/install/templates/create_collab_tables.rb.erb

@@ -0,0 +1,28 @@
+class CreateCollabTables < ActiveRecord::Migration[6.0]
+  def change
+    create_table :collab_documents<%= ", id: #{@primary_key_type.inspect}" if @primary_key_type %> do |t|
+      t.references :attached,      null: false, index: false, polymorphic: true<%= ", type: #{@primary_key_type.inspect}" if @primary_key_type %>
+      t.string     :attached_as,   null: false
+
+      t.index   [:attached_type, :attached_id, :attached_as], name: "index_collab_documents_on_attached"
+
+      t.jsonb   :content,          null: false
+      t.string  :schema_name,      null: false
+      t.integer :document_version, null: false, default: 0
+      t.text    :serialized_html
+
+      t.timestamps
+    end
+
+    create_table :collab_commits, id: false do |t|
+      t.references :document, null: false, foreign_key: {to_table: :collab_documents}, index: false<%= ", type: #{@primary_key_type.inspect}" if @primary_key_type %>
+      t.integer    :document_version, null: false
+      t.index      [:document_id, :document_version], unique: true, order: {document_version: :asc}, name: "index_collab_commits"
+
+      t.jsonb   :steps, array: true, null: false
+      t.string  :ref
+
+      t.datetime :created_at, null: false
+    end
+  end
+end

data/lib/generators/collab/install/templates/initializer.rb

@@ -4,27 +4,22 @@ Collab.config do |c|
   # To use a Git repo, see https://docs.npmjs.com/files/package.json#git-urls-as-dependencies
   c.schema_package = "prosemirror-schema-basic"
   # How many old transactions to keep per document
-  c.max_transactions = 250
+  c.max_commit_history_length = 250
+  # How many NodeJS child processes to run (shared among all threads)
+  c.num_js_processes = 3
 
-  # ActionCable settings
-  # ====================
   # The document channel to use for collaboration
-  # If you change this, you must pass {channel: "[ChannelName]"} as subscription params to the Javascript client
-  c.channel_name = "::CollabDocumentChannel"
+  # If you change this, you must pass the value as {channel: "[ChannelName]"} in the params from the ActionCable client
+  c.channel = "CollabDocumentChannel"
 
-  # ActionJob settings
-  # ==================
-  # The base job class to use
-  c.application_job = "::ApplicationJob"
-  # The job queue to use for DocumentTransaction jobs
-  c.queue_document_transaction_job_as = :default
+  # The class which jobs in the gem should inherit from
+  c.base_job = "ApplicationJob"
+  # The jobs to use, if you want to implement your own jobs
+  # c.commit_job = "..."
 
-
-  # ActiveRecord settings
-  # =====================
   # The class which models in the gem should inherit from
-  c.application_record = "::ApplicationRecord"
-  # If you want to use your own document model or document transaction model, 
-  c.document_model = "::Collab::Models::Document"
-  c.document_transaction_model = "::Collab::Models::DocumentTransaction"
+  c.base_record = "ApplicationRecord"
+  # The models to use, if you want to implement your own models
+  # c.document_model = "..."
+  # c.commit_model = "..."
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: collab
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Ben Aubin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2020-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -52,20 +52,20 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
-- app/jobs/collab/document_transaction_job.rb
+- app/jobs/collab/commit_job.rb
 - lib/collab.rb
-- lib/collab/bridge.rb
 - lib/collab/channel.rb
 - lib/collab/config.rb
+- lib/collab/engine.rb
 - lib/collab/has_collaborative_document.rb
+- lib/collab/js.rb
 - lib/collab/models/base.rb
+- lib/collab/models/commit.rb
 - lib/collab/models/document.rb
-- lib/collab/models/document_transaction.rb
-- lib/collab/railtie.rb
 - lib/collab/version.rb
 - lib/generators/collab/install/install_generator.rb
 - lib/generators/collab/install/templates/channel.rb
-- lib/generators/collab/install/templates/create_collab_tables.rb
+- lib/generators/collab/install/templates/create_collab_tables.rb.erb
 - lib/generators/collab/install/templates/initializer.rb
 - lib/tasks/collab_tasks.rake
 homepage: https://github.com/benaubin/rails-collab

data/app/jobs/collab/document_transaction_job.rb

@@ -1,7 +0,0 @@
-class Collab::DocumentTransactionJob < ::Collab.config.application_job.constantize
-  queue_as ::Collab.config.queue_document_transaction_job_as
-
-  def perform(document, data)
-    document.apply_transaction_now(data)
-  end
-end

data/lib/collab/bridge.rb

@@ -1,54 +0,0 @@
-require "collab/config"
-require "json"
-
-module Collab
-  class Bridge
-    class JSRuntimeError < StandardError
-      def initialize(data)
-        @js_backtrace = data["stack"].split("\n").map{|f| "JavaScript #{f.strip}"} if data["stack"]
-
-        super(data["name"] + ": " + data["message"])
-      end
-
-      def backtrace
-        return unless  val = super
-        
-        if @js_backtrace
-          @js_backtrace + val
-        else
-          val
-        end
-      end
-    end
-    
-    def initialize
-      @node = Dir.chdir(Rails.root) do
-        IO.popen(["node", "-e", "require('rails-collab-server')"], "r+")
-      end
-    end
-
-    def self.current
-      @current ||= new
-    end
-
-    def call(name, data = nil, schema_name:)
-      req = {name: name, data: data, schemaPackage: ::Collab.config.schema_package, schemaName: schema_name}
-      @node.puts(JSON.generate(req))
-      res = JSON.parse(@node.gets)
-      raise ::Collab::Bridge::JSRuntimeError.new(res["error"]) if res["error"]
-      res["result"]
-    end
-    
-    def apply_transaction(document, transaction, schema_name:)
-      call("applyTransaction", {doc: document, data: transaction}, schema_name: schema_name)
-    end
-
-    def html_to_document(html, schema_name:)
-      call("htmlToDoc", html, schema_name: schema_name)
-    end
-
-    def document_to_html(document, schema_name:)
-      call("docToHtml", document, schema_name: schema_name)
-    end
-  end
-end

data/lib/collab/models/document_transaction.rb

@@ -1,22 +0,0 @@
-module Collab
-  class Models::DocumentTransaction < ::Collab::Models::Base
-    belongs_to :document, class_name: ::Collab.config.document_model
-
-    validates :steps, presence: true
-    validates :document_version, presence: true
-
-    after_create_commit :broadcast
-
-    def as_json
-      {
-        v: document_version,
-        steps: steps,
-        ref: ref
-      }
-    end
-
-    def broadcast
-      ::Collab.config.channel_name.constantize.broadcast_to(document, as_json)
-    end
-  end
-end

data/lib/collab/railtie.rb

@@ -1,4 +0,0 @@
-module Collab
-  class Railtie < ::Rails::Railtie
-  end
-end

data/lib/generators/collab/install/templates/create_collab_tables.rb

@@ -1,32 +0,0 @@
-class CreateCollabTables < ActiveRecord::Migration[6.0]
-  def change
-    create_table :collab_documents, id: :uuid do |t|
-      t.references :attached, null: false, index: false, type: :uuid, polymorphic: true
-      t.string :attached_as
-      t.index [:attached_type, :attached_id, :attached_as], name: "index_collaborative_documents_on_attached"
-
-      t.string :schema_name, null: false
-
-      t.jsonb :document, null: false
-      t.integer :document_version, null: false, default: 0
-
-      t.text :serialized_html
-      t.integer :serialized_html_version
-
-      t.timestamps
-    end
-
-    create_table :collab_document_transactions, id: false do |t|
-      t.references :document, null: false, foreign_key: {to_table: :collab_documents}, type: :uuid, index: false
-
-      t.jsonb :steps, array: true, null: false
-      t.integer :document_version, null: false
-
-      t.string :ref
-
-      t.index [:document_id, :document_version], unique: true, order: {document_version: :asc}, name: "index_collaborative_document_transactions"
-
-      t.timestamps
-    end
-  end
-end