distribute_tree 0.0.1

data/.gitignore

@@ -0,0 +1,21 @@
+*.rbc
+*.sassc
+.sass-cache
+capybara-*.html
+.rspec
+.rvmrc
+/.bundle
+/vendor/bundle
+/log/*
+/tmp/*
+/db/*.sqlite3
+/public/system/*
+/coverage/
+/spec/tmp/*
+**.orig
+rerun.txt
+pickle-email-*.html
+.project
+config/initializers/secret_token.rb
+
+Gemfile.lock

data/Gemfile

@@ -0,0 +1 @@
+gemspec

data/Guardfile

@@ -0,0 +1,14 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :test do
+  watch(%r{^lib/(.+)\.rb$})     { |m| "test/#{m[1]}_test.rb" }
+  watch(%r{^test/.+_test\.rb$})
+  watch('test/test_helper.rb')  { "test" }
+
+  # Rails example
+  watch(%r{^app/models/(.+)\.rb$})                   { |m| "test/unit/#{m[1]}_test.rb" }
+  watch(%r{^app/controllers/(.+)\.rb$})              { |m| "test/functional/#{m[1]}_test.rb" }
+  watch(%r{^app/views/.+\.rb$})                      { "test/integration" }
+  watch('app/controllers/application_controller.rb') { ["test/functional", "test/integration"] }
+end

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 SunshineLibrary
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,76 @@
+distribute_tree
+===============
+用于实现单个Cloud和多个Local服务器之间数据共享的 Rails Engine 。
+
+Cloud(1) <-> Local(n) 同步机制概述
+---------------
+同步模式:
+
+* 自动: 保存一个资源均只同步自身的JSON和File，即不递归同步子集。
+* 手动: 新增一个学校，或者只想给某些学校，分配某些资源，在后台管理点击同步，并递归子集。
+
+启动方式
+---------------
+
+```bash
+RAILS_ENV=production bundle exec rake resque:work QUEUE='cloud_distribute_tree' --trace
+```
+
+如果需要对一个queue起多个work，那么多开几个rake进程就好了，系统会自动排队和并发处理的。
+
+[循环sync] local(只要在这里把资源标上创建地点即可让cloud识别) -> cloud -> locals
+
+同步架构变迁历史概述
+---------------
+#### 第一代(mysql replication)
+
+优点:
+
+1. 从数据库层面一致。
+
+缺点:
+
+1. 单向。
+1. 无法提供细力度控制和外部操作。
+
+#### 第二代(rabbitmq)
+
+优点:
+
+1. RabbitMQ高性能
+2. 引入基于JSON格式的API在不同系统间同步，并分成元数据和File两种数据类型分别分发。
+
+缺点：
+
+1. 按订阅全部同步，容易造成VPN网络堵塞。虽然可以改成点对点发送，但也失去了用rabbitmq的必要。
+2. 无细力度的UI，不透明。这个开发难度比 resque 高，且是另外一套系统。
+3. 无法管理细力度的失败的同步。
+
+#### 第三代(resque)
+架构：基于resque队列系统。队列可视化，rails风格。从优点上继承了上代的RabbitMQ架构。
+
+A. 初始化同步，或者中途中断后同步， cloud -> local。
+
+配置学校，可以按school同步，并在resque里递归从属或关联子集，产生新队列（这一想法基于之前给刘聪做的递归更新父级时间戳 [mongoid_touch_parents_recursively](https://github.com/SunshineLibrary/mongoid_touch_parents_recursively) 。
+
+B. 增量更新同步. cloud -> local (lesson) ，或者local -> cloud (piece) 。
+
+在数据同步后紧接着触发文件同步，都放在resque里。两者交互机制就是经典的client-server API 架构CRUD数据，这样也就符合了我来书屋面试前还没成型的想法，隐约地想要把学校local也看成是cloud的伪android客户端。删除也参考warpgate的做法。
+
+注: 感谢和小平的冲突，感谢和诺哥的讨论，在萨莉亚吃完晚饭后，萦绕的思绪突然激发了这一想法。这个完全不是时间逼的。
+
+FAQ
+---------------
+问题: paperclip包含的静态资源过大，比如100M，在cloud的resque连接上local的Rails接口去上传文件时会发生网络请求超时错误，并会占用一个unicorn slave。
+
+答案: 可以反过来做，改为在元数据同步过去到local后，不管paperclip包含的文件对象大小，
+      让local自己主动向cloud的nginx静态资源服务器读取大文件，并在下载完成后保存即可，从而达到异步的效果。
+      这点warpgate就是这样做的，跨网络传输文件还是下行快一点。
+
+TODO
+---------------
+1. 突破Mongoid.relations限制以支持ActiveModel
+
+相关引用
+---------------
+1. [warpgate](https://github.com/SunshineLibrary/warpgate)

data/app/assets/javascripts/distribute_tree.js

@@ -0,0 +1,137 @@
+// 添加分发按钮到资源*表格*的每一项，并显示同步状态。
+// =======================================
+//
+// 只在cloud开启该分发按钮
+//
+// =======================================
+// ### 使用方法 ###
+//
+// 在table加上属性model_name，以及在各tr的首个td里加上uuid和name两个属性。
+//
+// %table.{model_name: :App}
+//   %tr
+//     %th.span5 UUID
+//     %th.span3= t "App.name"
+//     ...
+//   - @apps.each_with_index do |app, idx|
+//     %tr
+//       %td{uuid: app.uuid, name: app.name}= app.uuid
+//       ...
+//
+// TODO 载入resque是否正常服务
+// TODO 加入验证用户
+//
+$(document).ready(function() {
+
+  var table = $("table[model_name]");
+  if (table.length === 0) { return false; } // 如果没有配置
+  if (table.length >= 2) { alert("目前distribute_tree.js只支持单页同步一种资源类型"); return false; }
+  var table_model_name = table.attr('model_name');
+
+  // Constant definement
+  var api_prefix = "/distribute_tree";
+
+  // 只有管理员才能同步学校
+  // TODO optimize
+  if (!$(".navbar ul.nav.pull-right li a").text().match(/陈大伟/) && (table_model_name === 'School')) { return false; }
+
+  // 1. 初始化页面元素
+  //
+  // 1.1 th 分发按钮
+  table.find('tr:first').append(
+    $("<th>").addClass("pointer span1").html(
+      $("<span>").addClass("icon icon-th")
+    )
+  // 并绑定 选择学校列表 弹框
+  ).on('click', 'th:last', function(event) {
+    var uuids = _.map(trs.find('td:last input:checked'), function(input) { return $(input).attr('uuid'); });
+    if (uuids.length === 0) { alert("请选择至少一个资源用于同步"); event.preventDefault(); return false; }
+    var modal_html = function() { return distribute_to_schools_modal.find('.modal'); };
+
+    function show() {
+      // 统计显示勾选的资源数目
+      modal_html().find('.modal-header h4.title span.count').html(uuids.length);
+      modal_html().modal('show');
+    }
+
+    // 判断是否ajax载入学校列表
+    if (_.isEmpty($("#distribute_to_schools_modal").html())) {
+      $.get(api_prefix + "/distribute/servers.html", function(html) {
+        distribute_to_schools_modal.html(html);
+
+        // 2. 绑定**提交事件**
+        distribute_to_schools_modal.on('click', 'button.submit', function() {
+          var school_uuids = _.map(distribute_to_schools_modal.find('table tr:gt(0) td input:checked'), function(input) { return $(input).attr('id'); });
+          if (school_uuids.length === 0) { alert("请选择至少一个学校用于同步"); event.preventDefault(); return false; }
+
+          // 2.1 处理元素显示
+          var btn = distribute_to_schools_modal.find(".modal-footer button.save-button");
+          btn.button('loading');
+
+          // 2.2 ajax提交数据
+          var data = {
+            model_name: table_model_name,
+            uuids: uuids,
+            school_uuids: school_uuids
+          };
+          // 2.3 服务器状态返回
+          $.post(api_prefix + '/distribute', data, function(e) {
+            var alert_html = modal_html().find(".alert");
+            alert_html.find('div.alert').removeClass('alert-success').removeClass('alert-error').addClass('alert-' + e.status);
+            alert_html.find('#flash_notice span.status').html('已经同步' + uuids.length + '个资源到' + school_uuids.length + '个服务器。');
+            $(".container.content").prepend(alert_html);
+          }).always(function(e) {
+            modal_html().modal('hide');
+            btn.button('reset');
+          });
+        });
+    // 并浮现
+        show();
+      });
+    } else {
+      show();
+    }
+  });
+  // 1.2 td 是否分发该资源
+  var trs = table.find('tr:gt(0)');
+  var is_defined_uuid_and_name_attrs = false; // check setup
+  _.each(trs, function(_tr) {
+    var tr = $(_tr);
+    var td_first = tr.find('td[uuid]');
+    if (td_first.length !== 0) { is_defined_uuid_and_name_attrs = true; }
+    tr.append(
+      $("<td>").append(
+        $("<input>").attr("type", "checkbox")
+                    .attr('uuid', td_first.attr('uuid'))
+                    .attr('name', td_first.attr('name'))
+      )
+    );
+  });
+  if (!is_defined_uuid_and_name_attrs) {
+    alert("没有定义在每行记录定义uuid和name，请察看distribute_tree.js的文档。");
+    event.preventDefault(); return false;
+  }
+  // 1.3 放置一个 学校列表容器
+  var distribute_to_schools_modal = $("body").append(
+    $("<div>").attr("id", "distribute_to_schools_modal")
+  ).find('#distribute_to_schools_modal');
+
+  // 1.4 载入各个资源的同步状态
+  var params = {model_name: table_model_name, uuids: _.map(trs.find('td:last input'), function(input) { return $(input).attr('uuid'); })};
+  $.get(api_prefix + "/distribute.json", params, function(result) {
+    _.each(trs, function(_tr) {
+      var td_last = $(_tr).find('td:last');
+      var urls    = result.data[td_last.find('input[type=checkbox]').attr('uuid')];
+      // 如果存在*没有分发完成*的记录
+      if (urls) {
+        td_last.append($("<div>").addClass('pointer server_urls').text(urls.length))
+               .append($("<div>").addClass('hide server_urls count').text(urls.join(', ')));
+      }
+    });
+    // 显示具体分发地址
+    trs.on('click', 'td:last .server_urls', function(e) {
+      $(e.target).closest('td').find('.server_urls.count').toggle();
+    });
+  });
+
+});

data/app/controllers/distribute_controller.rb

@@ -0,0 +1,105 @@
+# encoding: UTF-8
+class DistributeController < ApplicationController
+
+  #######################
+  #####  A. creator  ####
+  #######################
+
+  def create
+    error_messages = []
+    error_messages << "Missing model name" if params[:model_name].blank?
+    error_messages << "Missing uuid" if params[:uuids].blank?
+    error_messages << "Please select an school" if params[:school_uuids].blank?
+
+    if not error_messages.blank?
+      render json: {status: "error", messages: error_messages}, status: :bad_request; return false
+    end
+
+    server_urls = School.where(uuid: {"$in" => params[:school_uuids]}).map(&:server_url)
+
+    params[:uuids].to_a.each do |uuid|
+      item = params[:model_name].classify.safe_constantize.uuid(uuid)
+      item.distribute_with_children(server_urls)
+    end
+
+    render json: {status: "success", messages: ["已加入到发送队列"]}
+  end
+
+  def servers
+    @schools = School.asc(:_id).where(is_enable_sync: true)
+    render layout: false
+  end
+
+  def index
+    data = DistributeTreeStatus.where(
+      item_class: params[:model_name],
+      item_uuid:  {"$in" => params[:uuids]},
+    ).to_a.inject({}) do |h, status|
+      h[status.item_uuid] ||= []
+      h[status.item_uuid] << status.server_url
+      h
+    end
+
+    render json: {status: 'success', data: data }
+  end
+
+  # TODO
+  def paperclip_file
+    params[:model_name].constantize
+  end
+
+  #######################
+  #####  B. receiver ####
+  #######################
+
+  def receive
+    payload = params[:payload]
+
+    unless payload and payload[:model_name]
+      render json: {status: 'error', message: "payload and model_name required"}
+      return
+    end
+
+    model = payload[:model_name].classify().constantize
+
+    uuid = nil
+    if payload[:model] and payload[:model].is_a?(Hash)
+      uuid = payload[:model][:uuid]
+    end
+
+    unless model and uuid
+      render json: {status: 'error', message: "model and uuid not identified"}
+      return
+    end
+
+    model = model.unscoped # 兼容被删除资源
+
+    if payload[:model][:delete]
+      model.where(uuid: uuid).destroy_all
+      render json: {status: 'success', message: "model deleted"}
+      return
+    end
+
+    if model.where(uuid: uuid).exists?
+      # update, skip validation
+      item = model.uuid(uuid)
+    elsif [Piece, Image].include? model
+      # Local不自动创建Piece和Image, 只更新老师添加的
+      render json: {status: 'success'}
+      return
+    else
+      item = model.new
+    end
+
+    payload[:model].delete '_id'
+
+    item.assign_attributes payload[:model], without_protection: true
+    item.deleted_at = nil if payload[:model][:deleted_at].blank? # 恢复删除。也许assign_attributes payload[:model]这里面已经包含了。
+    if item.save validate: false
+      render json: {status: 'success'}
+    else
+      render json: {status: 'error', message: "fail to create or update model"}
+    end
+  end
+
+end

data/app/models/distribute_tree_status.rb

@@ -0,0 +1,16 @@
+# encoding: UTF-8
+
+class DistributeTreeStatus
+  include Mongoid::Document; self.store_in database: self.name.underscore
+  include Mongoid::Timestamps
+
+  field :item_class, type: String
+  field :item_uuid,  type: String
+  field :server_url, type: String
+
+  index({item_class: 1, item_uuid: 1, server_url: 1}, {background: true})
+
+  def self.insert item_class, item_uuid, server_url; DistributeTreeStatus.find_or_create_by(item_class: item_class, item_uuid: item_uuid, server_url: server_url) end
+  def self.delete item_class, item_uuid, server_url; DistributeTreeStatus.where(item_class: item_class, item_uuid: item_uuid, server_url: server_url).delete_all  end
+
+end

data/app/views/distribute/servers.html.haml

@@ -0,0 +1,38 @@
+.modal.hide.fade{ "aria-hidden" => "true", :role => "dialog", :tabindex => "-1"}
+  %ul.items
+
+  .modal-header
+    %button.close{"aria-hidden" => "true", "data-dismiss" => "modal", :type => "button"} ×
+    %h4.title
+      %span 分发
+      %span.count
+      %span 个资源到下列勾选服务器
+
+  .modal-body
+    = form_tag({controller: "distribute", action: "create"}, method: "post", class: "form-horizontal") do
+      %div
+        %table.table.table-striped
+          %tbody
+            %tr
+              %th.span5.uuid.hide UUID
+              %th.span3 服务器
+              // %th.span2 服务器状态
+              %th.span2 分发
+            - @schools.each do |school|
+              %tr
+                %td.uuid.hide= school.uuid
+                %td= link_to school.name, school_path(school.uuid)
+                %td= check_box_tag(school.uuid)
+
+  .modal-footer
+    %button.submit.btn.btn-primary{:class => 'save-button', "data-loading-text" => "分发中…" } 分发
+    %button.btn{"aria-hidden" => "true", "data-dismiss" => "modal"} 取消
+
+  .alert.hide
+    %div.alert
+      %a.close{'data-dismiss' => 'alert'} x
+      %div#flash_notice
+        %span.status
+        %span 请前往
+        = link_to "同步队列", "/resque/queues/#{Mongoid::DistributeTree.default_queue}"
+        %span 察看最新状态

data/config/routes.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+Rails.application.routes.draw do
+
+  namespace :distribute_tree do
+    resources :distribute do
+      get  :servers, :paperclip_file
+      post :receive
+    end
+  end
+
+end