collab 0.1.9 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d18b58e0a43c89ef45deb94871423015a54a755cd565ef86e9025bf5ca50e7a
-  data.tar.gz: 805a04e74c43e4d2deb0514be61ccce650d2952bac9538f1d1d9721ef9e71678
+  metadata.gz: 714939d140c87683e4c7adf35556a020acb7697ce4fb421dadaffb76d2b7c15f
+  data.tar.gz: bd24f1cb2b7d137f97f4c66636703f365467be4d915cd1498a1dabc72b958abd
 SHA512:
-  metadata.gz: bd446fb6609196d039e53f18387f95da8fbc416a96b2e3e59a18ada3468bcf88957cb1b77b7e49debe0bb4691acc5182a952c9abeceab9e69a07528e8977ba54
-  data.tar.gz: da896456dce781c41236d76cc914cb21c27e565ba269e0093d8de8d5f05e0a08685af6674a5791753cd94a8499b9274efb8a21bd790b139f4b337975d3fdd475
+  metadata.gz: c6f3612a1ad7908d16aa06bccba6c48a87178c6be1aed5824233cdf34a67dc389353b97956167bdce4eb6b4a4e9621910b16e39677963f9dd9a70472d54edc04
+  data.tar.gz: '028b45335893ab9d6d59936d9306abe5e5631e635532a2e28e660e4c9b43c674ad2999cc33580e6d997b5ebf88b8ce2e5120cd99d8642a0950b9440a99faa09f'

data/app/jobs/collab/commit_job.rb

@@ -0,0 +1,7 @@
+class Collab::CommitJob < ::Collab.config.base_job.constantize
+  queue_as :default
+
+  def perform(document, data)
+    document.commits.from_json(data).apply!
+  end
+end

data/lib/collab.rb

@@ -1,19 +1,15 @@
-require "collab/railtie"
+require "collab/version"
+require "collab/config"
+require "collab/js"
+require "collab/engine"
 
 module Collab
-  def self.config
-    @config ||= Collab::Config.new
-    return @config unless block_given?
-    yield @config
-  end
-
-  autoload "Config", "collab/config"
-  autoload "Bridge", "collab/bridge"
+  autoload "Channel", "collab/channel"
   autoload "HasCollaborativeDocument", "collab/has_collaborative_document"
   
   module Models
     autoload "Base", "collab/models/base"
     autoload "Document", "collab/models/document"
-    autoload "DocumentTransaction", "collab/models/document_transaction"
+    autoload "Commit", "collab/models/commit"
   end
 end

data/lib/collab/channel.rb

@@ -0,0 +1,33 @@
+module Collab
+  module Channel
+    def document; @document end
+
+    def subscribed
+      @document = find_document
+
+      starting_version = params[:startingVersion]&.to_i
+      raise "missing startingVersion" if starting_version.nil?
+
+      stream_for document
+      
+      commits = document.commits
+                        .where("document_version > ?", starting_version)
+                        .order(document_version: :asc)
+                        .load
+              
+      unless commits.empty?
+        raise "invalid version" unless commits.first.document_version == (starting_version + 1)
+        commits.lazy.map(&:as_json).each(method(:transmit))
+      end
+    end
+
+    def commit(data)
+      authorize_commit!(data)
+      document.commit_later(data)
+    end
+
+    def unsubscribed
+      stop_all_streams # this may not be needed
+    end
+  end
+end

data/lib/collab/config.rb

@@ -1,15 +1,34 @@
 module Collab
-  class Config
-    attr_accessor :max_transactions, :application_job, :queue_document_transaction_job_as, :schema_package, :application_record, :document_transaction_model, :document_model
-    
-    def find_document_for_subscribe(&block)
-      @find_document_for_subscribe unless block
-      @find_document_for_subscribe = block
+  @config_mutex = Mutex.new
+
+  def self.config
+    if block_given?
+      @config_mutex.synchronize do
+        @config ||= ::Collab::Config.new
+        raise "[Collab] Tried to configure gem after first use" if @config.frozen?
+        yield @config
+      end
+    else
+      raise "[Collab] Missing configuration - Have you run `rails g collab:install` yet?" unless @config
+      @config.freeze # really weird stuff could happen if the config changes after first use, so freeze config
     end
+  end
+
+  class Config
+    attr_accessor :base_record,
+                  :base_job,
+                  :channel,
+                  :commit_job,
+                  :commit_model,
+                  :document_model,
+                  :max_commit_history_length,
+                  :num_js_processes,
+                  :schema_package
 
-    def authorize_update_document(&block)
-      @authorize_update_document unless block
-      @authorize_update_document = block
+    def initialize
+      self.commit_job = "Collab::CommitJob"
+      self.document_model = "Collab::Models::Document"
+      self.commit_model = "Collab::Models::Commit"
     end
   end
 end

data/lib/collab/engine.rb

@@ -0,0 +1,6 @@
+if defined?(Rails)
+  module Collab
+      class Engine < ::Rails::Engine
+      end
+  end
+end

data/lib/collab/js.rb

@@ -0,0 +1,93 @@
+require "json"
+
+module Collab
+  module JS
+    @queue = Queue.new
+    @queue_initialized = false
+    @queue_initialization_mutex = Mutex.new
+
+    class <<self
+      def queue
+        initialize_queue unless @queue_initialized
+        @queue
+      end
+
+      # Calls the block given with a JS process acquired from the queue
+      # Will block until a JS process is available
+      def with_js
+        js = queue.pop
+        yield js
+      ensure
+        queue << js
+      end
+
+      def call(name, data = nil, schema_name:)
+        with_js { |js| js.call(name, *arguments, &block) }
+      end
+      
+      def apply_commit(document, commit, schema_name:)
+        call("applyCommit", {doc: document, commit: commit}, schema_name: schema_name)
+      end
+
+      def html_to_document(html, schema_name:)
+        call("htmlToDoc", html, schema_name: schema_name)
+      end
+
+      def document_to_html(document, schema_name:)
+        call("docToHtml", document, schema_name: schema_name)
+      end
+
+      private 
+      # Thread-safe initialization of the NodeJS process queue
+      def initialize_queue
+        @queue_initialization_mutex.synchronize do
+          unless @queue_initialized
+            ::Collab.config.num_js_processes.times { @queue << ::Collab::JS::JSProcess.new }
+            @queue_initialized = true
+          end
+        end
+      end
+    end
+
+    class JSProcess
+      def initialize
+         @node = if defined?(Rails)
+                   Dir.chdir(Rails.root) { open_node }
+                 else
+                   open_node
+                 end
+      end
+
+      def call(name, data = nil, schema_name:)
+        req = {name: name, data: data, schemaPackage: ::Collab.config.schema_package, schemaName: schema_name}
+        @node.puts(JSON.generate(req))
+        res = JSON.parse(@node.gets)
+        raise ::Collab::JS::JSRuntimeError.new(res["error"]) if res["error"]
+        res["result"]
+      end
+
+      private
+      def open_node
+        IO.popen(["node", "-e", "require('rails-collab-server')"], "r+")
+      end
+    end
+
+    class JSRuntimeError < StandardError
+      def initialize(data)
+        @js_backtrace = data["stack"].split("\n").map{|f| "JavaScript #{f.strip}"} if data["stack"]
+
+        super(data["name"] + ": " + data["message"])
+      end
+
+      def backtrace
+        return unless  val = super
+        
+        if @js_backtrace
+          @js_backtrace + val
+        else
+          val
+        end
+      end
+    end
+  end
+end

data/lib/collab/models/base.rb

@@ -1,5 +1,5 @@
 module Collab
-  class Models::Base < ::Collab.config.application_record.constantize
+  class Models::Base < ::Collab.config.base_record.constantize
     self.abstract_class = true
     self.table_name_prefix = 'collab_'
   end

data/lib/collab/models/commit.rb

@@ -0,0 +1,53 @@
+module Collab
+  class Models::Commit < ::Collab::Models::Base
+    belongs_to :document, class_name: ::Collab.config.document_model
+
+    validates :steps, presence: true
+    validates :document_version, presence: true
+    validates :ref, length: { maximum: 36 }
+
+    after_create_commit :broadcast
+
+    def self.from_json(data)
+      new(document_version: data["v"]&.to_i, steps: data["steps"], ref: data["ref"])
+    end
+
+    def as_json
+      {
+        v: document_version,
+        steps: steps,
+        ref: ref
+      }
+    end
+
+    def apply_later
+      raise "cannot apply persisted commit" if self.persisted?
+      raise "commit not valid" unless self.valid?
+      return false if self.document.document_version != self.document_version
+
+      ::Collab.config.commit_job.constantize.perform_later(self.document, as_json)
+    end
+
+    def apply!
+      raise "cannot apply persisted commit" if self.persisted?
+      raise "commit not valid" unless self.valid?
+      return false if self.document.document_version != self.document_version # optimization, prevents need to apply lock if outdated
+
+      self.document.with_lock do
+        return false if self.document.document_version != self.document_version
+        
+        return false unless result = ::Collab::JS.apply_commit(self.document, self.to_json, schema_name: self.document.schema_name)
+        
+        self.document.document = result["doc"]
+        self.document.document_version = self.document_version
+        
+        self.document.save!
+        self.save!
+      end
+    end
+
+    def broadcast
+      ::Collab.config.channel_name.constantize.broadcast_to(document, as_json)
+    end
+  end
+end

data/lib/collab/models/document.rb

@@ -1,73 +1,52 @@
 module Collab
   class Models::Document < ::Collab::Models::Base
     belongs_to :attached, polymorphic: true
-    has_many :transactions, class_name: ::Collab.config.document_transaction_model, foreign_key: :document_id
+    has_many :commits, class_name: ::Collab.config.commit_model, foreign_key: :document_id
 
-    validates :document, presence: true
+    validates :content, presence: true
     validates :document_version, presence: true, numericality: {only_integer: true, greater_than_or_equal_to: 0}
     validates :schema_name, presence: true
 
-    before_save :nullify_serialized_html, unless: :serialized_html_fresh?
-    after_save :delete_old_transactions
+    after_save :delete_old_commits
 
-    # The already-serialized html version of this document
-    def serialized_html
-      return super if serialized_html_fresh?
-    end
-
-    def serialized_html_fresh?
-      serialized_html_version == document_version
-    end
-
-    # Serialize the document to html - will be cached if possible (somewhat expensive)
+    # Serialize the document to html, uses cached if possible.
+    # Note that this may lock the document
     def to_html
       return serialized_html if serialized_html
 
-      serialized_html = ::Collab::Bridge.current.document_to_html(self.document, schema_name: schema_name)
-      self.update! serialized_html: serialized_html, serialized_html_version: document_version
-      serialized_html
+      serialized_version = self.document_version
+      ::Collab::JS.document_to_html(self.content, schema_name: schema_name).tap do |serialized_html|
+        Thread.new do # use a thread to prevent deadlocks and avoid incuring the cost of an inline-write
+          self.with_lock do
+            self.update_attribute(:serialized_html, serialized_html) if serialized_version == self.version and self.serialized_html.nil?
+          end
+        end
+      end
     end
 
     def from_html(html)
-      self.document = ::Collab::Bridge.current.html_to_document(html, schema_name: schema_name)
-    end
-
-    def perform_transaction_later(data)
-      ::Collab::DocumentTransactionJob.perform_later(@document, data)
+      self.content = ::Collab::JS.html_to_document(html, schema_name: schema_name)
     end
 
-    def apply_transaction_now(data)
-      return unless data["v"].is_a?(Integer)
-
-      return if (data["v"] - 1) != self.document_version # if expired between queue and perform, there is no need to aquire a lock. this is an optimization - not a guarantee
-      with_lock do
-        return if (data["v"] - 1) != self.document_version # now that we've aquired a lock to the record, ensure that we're still accessing the correct version
-        
-        transaction_result = ::Collab::Bridge.current.apply_transaction(self.document, data, schema_name: self.schema_name)
-        return unless transaction_result # check to make sure the transaction succeeded
-        
-        self.document = transaction_result["doc"]
-        self.document_version = data["v"]
-        save!
-
-        transactions.create! document_version: self.document_version, steps: data["steps"], ref: data["ref"]
-      end
+    def commit_later(data)
+      commits.from_json(data).apply_later
     end
 
     def as_json
-      {id: id, document: document, version: self.document_version}
+      {id: id, content: content, version: document_version}
     end
 
-    private
-
-    def nullify_serialized_html
+    def content_will_change!
+      super
       self.serialized_html = nil
     end
 
-    def delete_old_transactions
-      cutoff = document_version - ::Collab.config.max_transactions
+    private
+
+    def delete_old_commits
+      cutoff = document_version - ::Collab.config.max_commit_history_length
       return if cutoff <= 0
-      transactions.where("document_version < ?", cutoff).delete_all
+      commits.where("document_version < ?", cutoff).delete_all
     end
   end
 end

data/lib/collab/version.rb

@@ -1,3 +1,3 @@
 module Collab
-  VERSION = '0.1.9'
+  VERSION = '0.3.0'
 end

data/lib/generators/collab/install/install_generator.rb

@@ -17,11 +17,15 @@ module Collab
     end
 
     def copy_files
+      @primary_key_type = Rails.application.config.generators.active_record[:primary_key_type]
+
       migration_template(
-        "create_collab_tables.rb",
+        "create_collab_tables.rb.erb",
         "db/migrate/create_collab_tables.rb",
       )
       
+      copy_file "channel.rb", "app/channels/collab_document_channel.rb"
+
       copy_file "initializer.rb", "config/initializers/collab.rb"
     end
   end

data/lib/generators/collab/install/templates/channel.rb

@@ -0,0 +1,22 @@
+class CollabDocumentChannel < ApplicationCable::Channel
+  include Collab::Channel
+
+  private
+
+  # Find the document to subscribe to based on the params passed to the channel
+  # Authorization may also be performed here (raise an error to prevent subscription)
+  def find_document
+    Collab::Models::Document.find(params[:document_id]).tap do |document|
+      # TODO: Replace with your own authorization logic
+      raise "authorization not implemented"
+    end
+  end
+
+  # Called a commit is first received for processing
+  # Throw an error to prevent the commit from being processed
+  # You should consider adding some type of rate-limiting here
+  def authorize_commit!(data)
+    # TODO: Replace with your own authorization logic
+    raise "authorization not implemented"
+  end
+end

data/lib/generators/collab/install/templates/create_collab_tables.rb.erb

@@ -0,0 +1,28 @@
+class CreateCollabTables < ActiveRecord::Migration[6.0]
+  def change
+    create_table :collab_documents<%= ", id: #{@primary_key_type.inspect}" if @primary_key_type %> do |t|
+      t.references :attached,      null: false, index: false, polymorphic: true<%= ", type: #{@primary_key_type.inspect}" if @primary_key_type %>
+      t.string     :attached_as,   null: false
+
+      t.index   [:attached_type, :attached_id, :attached_as], name: "index_collab_documents_on_attached"
+
+      t.jsonb   :content,          null: false
+      t.string  :schema_name,      null: false
+      t.integer :document_version, null: false, default: 0
+      t.text    :serialized_html
+
+      t.timestamps
+    end
+
+    create_table :collab_commits, id: false do |t|
+      t.references :document, null: false, foreign_key: {to_table: :collab_documents}, index: false<%= ", type: #{@primary_key_type.inspect}" if @primary_key_type %>
+      t.integer    :document_version, null: false
+      t.index      [:document_id, :document_version], unique: true, order: {document_version: :asc}, name: "index_collab_commits"
+
+      t.jsonb   :steps, array: true, null: false
+      t.string  :ref
+
+      t.datetime :created_at, null: false
+    end
+  end
+end

data/lib/generators/collab/install/templates/initializer.rb

@@ -4,36 +4,22 @@ Collab.config do |c|
   # To use a Git repo, see https://docs.npmjs.com/files/package.json#git-urls-as-dependencies
   c.schema_package = "prosemirror-schema-basic"
   # How many old transactions to keep per document
-  c.max_transactions = 250
+  c.max_commit_history_length = 250
+  # How many NodeJS child processes to run (shared among all threads)
+  c.num_js_processes = 3
 
-  # Handlers
-  # ========
-  # Find a the document to subscribe to based on the params passed to the channel
-  # Authorization may also be performed here (raise an error)
-  # The block is executed in the scope of the ActionCable channel within #subscribe
-  c.find_document_for_subscribe do
-    Collab::Models::Document.find params[:document_id]
-  end
-  # Called when a client submits a transaction in order to update a document
-  # You should throw an error if unauthorized
-  # The block is executed in the instance of the channel
-  c.authorize_update_document do |document, transaction_data|
-    # raise "authorization failed"
-  end
+  # The document channel to use for collaboration
+  # If you change this, you must pass the value as {channel: "[ChannelName]"} in the params from the ActionCable client
+  c.channel = "CollabDocumentChannel"
 
-  # ActionJob settings
-  # ==================
-  # The base job class to use
-  c.application_job = "::ApplicationJob"
-  # The job queue to use for DocumentTransaction jobs
-  c.queue_document_transaction_job_as = :default
+  # The class which jobs in the gem should inherit from
+  c.base_job = "ApplicationJob"
+  # The jobs to use, if you want to implement your own jobs
+  # c.commit_job = "..."
 
-
-  # ActiveRecord settings
-  # =====================
   # The class which models in the gem should inherit from
-  c.application_record = "::ApplicationRecord"
-  # If you want to use your own document model or document transaction model, 
-  c.document_model = "::Collab::Models::Document"
-  c.document_transaction_model = "::Collab::Models::DocumentTransaction"
+  c.base_record = "ApplicationRecord"
+  # The models to use, if you want to implement your own models
+  # c.document_model = "..."
+  # c.commit_model = "..."
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: collab
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.3.0
 platform: ruby
 authors:
 - Ben Aubin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-30 00:00:00.000000000 Z
+date: 2020-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -52,19 +52,20 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
-- app/channels/collab/document_channel.rb
-- app/jobs/collab/document_transaction_job.rb
+- app/jobs/collab/commit_job.rb
 - lib/collab.rb
-- lib/collab/bridge.rb
+- lib/collab/channel.rb
 - lib/collab/config.rb
+- lib/collab/engine.rb
 - lib/collab/has_collaborative_document.rb
+- lib/collab/js.rb
 - lib/collab/models/base.rb
+- lib/collab/models/commit.rb
 - lib/collab/models/document.rb
-- lib/collab/models/document_transaction.rb
-- lib/collab/railtie.rb
 - lib/collab/version.rb
 - lib/generators/collab/install/install_generator.rb
-- lib/generators/collab/install/templates/create_collab_tables.rb
+- lib/generators/collab/install/templates/channel.rb
+- lib/generators/collab/install/templates/create_collab_tables.rb.erb
 - lib/generators/collab/install/templates/initializer.rb
 - lib/tasks/collab_tasks.rake
 homepage: https://github.com/benaubin/rails-collab

data/app/channels/collab/document_channel.rb

@@ -1,28 +0,0 @@
-class Collab::DocumentChannel < ApplicationCable::Channel
-  def subscribed
-    @document = instance_exec(::Collab.config.find_document_for_subscribe)
-
-    starting_version = params[:startingVersion]&.to_i
-    raise "missing startingVersion" if starting_version.nil?
-
-    stream_for @document
-    
-    transactions = @document.transactions
-                            .where("document_version > ?", starting_version)
-                            .order(document_version: :asc)
-                            .load
-            
-    raise "invalid version" unless transactions.first.document_version == (starting_version + 1) unless transactions.empty?
-
-    transactions.lazy.map(&:as_json).each(method(:transmit))
-  end
-
-  def submit(data)
-    instance_exec(@document, data, &::Collab.config.authorize_update_document)
-    @document.perform_transaction_later(data)
-  end
-
-  def unsubscribed
-    stop_all_streams # this may not be needed
-  end
-end

data/app/jobs/collab/document_transaction_job.rb

@@ -1,7 +0,0 @@
-class Collab::DocumentTransactionJob < ::Collab.config.application_job.constantize
-  queue_as ::Collab.config.queue_document_transaction_job_as
-
-  def perform(document, data)
-    document.apply_transaction_now(data)
-  end
-end

data/lib/collab/bridge.rb

@@ -1,54 +0,0 @@
-require "collab/config"
-require "json"
-
-module Collab
-  class Bridge
-    class JSRuntimeError < StandardError
-      def initialize(data)
-        @js_backtrace = data["stack"].split("\n").map{|f| "JavaScript #{f.strip}"} if data["stack"]
-
-        super(data["name"] + ": " + data["message"])
-      end
-
-      def backtrace
-        return unless  val = super
-        
-        if @js_backtrace
-          @js_backtrace + val
-        else
-          val
-        end
-      end
-    end
-    
-    def initialize
-      @node = Dir.chdir(Rails.root) do
-        IO.popen(["node", "-e", "require('rails-collab-server')"], "r+")
-      end
-    end
-
-    def self.current
-      @current ||= new
-    end
-
-    def call(name, data = nil, schema_name:)
-      req = {name: name, data: data, schemaPackage: ::Collab.config.schema_package, schemaName: schema_name}
-      @node.puts(JSON.generate(req))
-      res = JSON.parse(@node.gets)
-      raise ::Collab::Bridge::JSRuntimeError.new(res["error"]) if res["error"]
-      res["result"]
-    end
-    
-    def apply_transaction(document, transaction, schema_name:)
-      call("applyTransaction", {doc: document, data: transaction}, schema_name: schema_name)
-    end
-
-    def html_to_document(html, schema_name:)
-      call("htmlToDoc", html, schema_name: schema_name)
-    end
-
-    def document_to_html(document, schema_name:)
-      call("docToHtml", document, schema_name: schema_name)
-    end
-  end
-end

data/lib/collab/models/document_transaction.rb

@@ -1,22 +0,0 @@
-module Collab
-  class Models::DocumentTransaction < ::Collab::Models::Base
-    belongs_to :document, class_name: ::Collab.config.document_model
-
-    validates :steps, presence: true
-    validates :document_version, presence: true
-
-    after_create_commit :broadcast
-
-    def as_json
-      {
-        v: document_version,
-        steps: steps,
-        ref: ref
-      }
-    end
-
-    def broadcast
-      ::Collab::DocumentChannel.broadcast_to(document, as_json)
-    end
-  end
-end

data/lib/collab/railtie.rb

@@ -1,4 +0,0 @@
-module Collab
-  class Railtie < ::Rails::Railtie
-  end
-end

data/lib/generators/collab/install/templates/create_collab_tables.rb

@@ -1,32 +0,0 @@
-class CreateCollabTables < ActiveRecord::Migration[6.0]
-  def change
-    create_table :collab_documents, id: :uuid do |t|
-      t.references :attached, null: false, index: false, type: :uuid, polymorphic: true
-      t.string :attached_as
-      t.index [:attached_type, :attached_id, :attached_as], name: "index_collaborative_documents_on_attached"
-
-      t.string :schema_name, null: false
-
-      t.jsonb :document, null: false
-      t.integer :document_version, null: false, default: 0
-
-      t.text :serialized_html
-      t.integer :serialized_html_version
-
-      t.timestamps
-    end
-
-    create_table :collab_document_transactions, id: false do |t|
-      t.references :document, null: false, foreign_key: {to_table: :collab_documents}, type: :uuid, index: false
-
-      t.jsonb :steps, array: true, null: false
-      t.integer :document_version, null: false
-
-      t.string :ref
-
-      t.index [:document_id, :document_version], unique: true, order: {document_version: :asc}, name: "index_collaborative_document_transactions"
-
-      t.timestamps
-    end
-  end
-end