lcms-engine 0.2.0 → 0.3.0

data/config/locales/admin/en.yml

@@ -20,6 +20,7 @@ en:
             success: Access Code updated successfully.
         common:
           editing_disabled: Resource editing is disabled. Turn editing on in order to edit resources.
+          empty_folder: Google Documents folder is empty
         curriculums:
           update:
             success: Curriculum Tree saved successfully.

data/config/routes.rb

@@ -1,37 +1,8 @@
 # frozen_string_literal: true
 
 Lcms::Engine::Engine.routes.draw do
-  get '/404'          => 'pages#not_found'
-  get '/about'        => 'pages#show_slug', slug: 'about'
-  get '/about/people' => 'pages#show_slug', slug: 'about_people'
-  get '/tos'          => 'pages#show_slug', slug: 'tos',     as: :tos_page
-  get '/privacy'      => 'pages#show_slug', slug: 'privacy', as: :privacy_page
-  get '/forthcoming'  => 'pages#forthcoming'
-
-  get '/search' => 'search#index'
-
   mount PdfjsViewer::Rails::Engine, at: '/pdfjs', as: 'pdfjs'
 
-  resources :downloads, only: [:show] do
-    member do
-      get :preview
-      get 'pdf_proxy(/:s3)' => 'downloads#pdf_proxy', as: :pdf_proxy_download, constraints: { s3: %r{[^\/]+} }
-    end
-  end
-
-  resources :explore_curriculum, only: %i(index show)
-  resources :enhance_instruction, only: :index
-  resources :find_lessons, only: :index
-  resources :pages, only: :show
-  resources :resources, only: :show do
-    get :pdf_proxy, on: :collection, path: 'pdf-proxy'
-  end
-  resource :survey, only: %i(create show)
-
-  get '/resources/:id/related_instruction' => 'resources#related_instruction', as: :related_instruction
-  get '/media/:id' => 'resources#media', as: :media
-  get '/other/:id' => 'resources#generic', as: :generic
-
   resources :documents, only: :show do
     member do
       post 'export', to: 'documents#export'
@@ -45,12 +16,12 @@ Lcms::Engine::Engine.routes.draw do
       get 'preview/gdoc', to: 'materials#preview_gdoc'
     end
   end
+  resources :resources, only: [:show]
 
   unless ENV.fetch('DEVISE_ROUTES_REDEFINED', false)
     devise_for :users, class_name: 'Lcms::Engine::User',
                        controllers: {
-                         registrations: 'lcms/engine/registrations',
-                         sessions: 'lcms/engine/sessions'
+                         registrations: 'lcms/engine/registrations'
                        },
                        module: :devise
   end
@@ -62,9 +33,7 @@ Lcms::Engine::Engine.routes.draw do
   namespace :admin do
     get '/' => 'welcome#index'
     get '/whoami' => 'admin#whoami'
-    get 'google_oauth2_callback' => 'google_oauth2#callback'
     get '/association_picker' => 'association_picker#index'
-    resources :reading_assignment_texts
     resource :resource_bulk_edits, only: %i(new create)
     get '/resource_picker' => 'resource_picker#index'
     resources :resources, except: :show do
@@ -73,7 +42,6 @@ Lcms::Engine::Engine.routes.draw do
         post :bundle
       end
     end
-    resources :pages, except: :show
     resources :settings, only: [] do
       patch :toggle_editing_enabled, on: :collection
     end
@@ -106,8 +74,6 @@ Lcms::Engine::Engine.routes.draw do
     resource :batch_reimport, only: %i(new create)
   end
 
-  get '/ExcludeMeGA' => 'analytics_exclusion#exclude'
-
   get '/*slug' => 'resources#show', as: :show_with_slug
 
   root to: 'welcome#index'

data/db/migrate/20200422130652_add_reimported_at_to_documents.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class AddReimportedAtToDocuments < ActiveRecord::Migration[5.2]
+class AddReimportedAtToDocuments < Lcms::Engine::Migration
   def change
     add_column :documents, :reimported_at, :datetime
   end

data/db/migrate/20200429130353_drop_pages.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropPages < Lcms::Engine::Migration
+  def change
+    drop_table :pages do |t|
+      t.text :body, null: false
+      t.string :slug, null: false
+      t.string :title, null: false
+
+      t.timestamps null: false
+    end
+  end
+end

data/docs/env-variables.md

@@ -0,0 +1,87 @@
+## Required variables
+
+- APP_NAME
+- APPLICATION_DOMAIN
+- AWS_ACCESS_KEY_ID
+- AWS_REGION
+- AWS_S3_BUCKET_NAME
+- AWS_SECRET_ACCESS_KEY
+- BACKGROUND_JOBS
+- ELASTICSEARCH_ADDRESS
+- NODE_ENV
+- POSTGRESQL_ADDRESS
+- POSTGRESQL_DATABASE
+- POSTGRESQL_USERNAME
+- POSTGRESQL_PASSWORD
+- POSTGRESQL_PORT
+- WEBPACKER_CHECK_YARN
+- WKHTMLTOPDF_PATH
+
+## All other variables
+
+* [AirBrake](#airbrake-config)
+* [Amazon](#amazon-config)
+* [Google](#google-config)
+* [Miscellaneous settings](#miscellaneous-settings)
+* [PDF](#pdf-related-config)
+* [Postgres](#postgres-config)
+
+### AirBrake config
+The project is set up to support Airbrake, if you use it. You can provide an Airbrake ID and key in the config to enable it (Should be omitted for `development`&`test` environments)
+
+|Name|Description|
+|----|-----------|
+|AIR_BRAKE_PROJECT_ID|Project id in the AirBrake service|
+|AIR_BRAKE_PROJECT_KEY|Project key in the AirBrake service|
+
+### Amazon config
+The project uses AWS S3 to store and serve generated PDFs of learning content. You will need to set up an S3 bucket and provide a key and secret here:
+
+|Name|Description|
+|----|-----------|
+|AWS_ACCESS_KEY_ID|Public part of the AWS credentials|
+|AWS_REGION|Name of the region used for AWS service|
+|AWS_SECRET_ACCESS_KEY|Private part of the AWS credentials|
+|AWS_S3_BUCKET_NAME|Bucket used by application to store generated files|
+
+### Google config
+The project uses several Google products, including analytics, OAuth for allowing the application to use Google APIs, and several Google Drive folders.
+
+|Name|Description|
+|----|-----------|
+|GOOGLE_APPLICATION_FOLDER_ID|Id of the Google Drive folder where generated lessons and materials will be placed(It's `0B7` for url like `https://drive.google.com/drive/u/0/folders/0B7/...`)|
+|GOOGLE_APPLICATION_SCRIPT_ID|Id of the Google Script created to post-process generated Google documents. More details can be found [here](google-cloud-platform-setup.md)|
+|GOOGLE_APPLICATION_SCRIPT_FUNCTION|Name of the function to call to start post-processing|
+|GOOGLE_APPLICATION_TEMPLATE_PORTRAIT|Id of the Google document which is a template for portrait materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)|
+|GOOGLE_APPLICATION_TEMPLATE_LANDSCAPE|Id of the Google document which is a template for landscape materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)|
+|GOOGLE_APPLICATION_PREVIEW_FOLDER_ID| Folder ID where preview documents should get placed
+|GOOGLE_API_CLIENT_UPLOAD_RETRIES||
+|GOOGLE_API_CLIENT_UPLOAD_TIMEOUT||
+
+### Miscellaneous settings
+|Name|Description|
+|----|-----------|
+|APP_NAME|The title which will be shown in the page title and in other places. Something like `OpenSciEd LCMS`|
+|BITLY_API_TOKEN|Token of the Bitly service|
+|RESQUE_NAMESPACE|Value is used to separate data stored in Redis when the same redis server is used by multiple environments|
+|PUPPETEER_TIMEOUT|Default is 30 sec|
+|WORKERS_COUNT|Used on the servers. Specifies the number of workers to be started|
+|ELASTICSEARCH_ADDRESS|Elasticsearch server address|
+
+Obs: if you're setting a local dev machine on OSX and getting small fonts when generating pdfs, try downgrading wkhtmltopdf to `0.12.3`
+
+### PDF related config
+|Name|Description|
+|----|-----------|
+|ENABLE_BASE64_CACHING|Turns on/off caching of assets used for PDF generation (`true` by default, recommended as `false` for local env)|
+|NODE_ENV|We're using puppeteer(npm package) for PDF Generation, should be set as `production` for other env than local|
+|WKHTMLTOPDF_PATH|Path to the [wkhtmltopdf](https://wkhtmltopdf.org) binary. Default to `/usr/local/bin/wkhtmltopdf`. Will be removed when we will be certain on puppeteer|
+
+### Postgres config
+|Name|
+|----|
+|POSTGRESQL_ADDRESS|
+|POSTGRESQL_DATABASE|
+|POSTGRESQL_USERNAME|
+|POSTGRESQL_PASSWORD|
+|POSTGRESQL_PORT|

data/docs/google-cloud-platform-setup.md

@@ -0,0 +1,106 @@
+An application calls the Google Application Script in order to post-process generated lesson materials. Instructions below contains all the necessary steps need to set up the integration.
+
+All environment variables and user names provided here have been set up to automate deployment on [Cloud66](https://www.cloud66.com) service stacks. If you plan to use another service provider please change all the variables accordingly.
+
+1. [Set up](#set-up)
+2. [How to update Google Application Script](#how-to-update-google-application-script)
+
+## Set up
+
+1. Login to google (all google docs will be saved under this account)
+
+2. Create project (i.e. LCMS-dev) at [google cloud](https://console.cloud.google.com)
+
+3. Enable Google Drive API, Google Apps Script Execution API for that [project](https://console.cloud.google.com/apis/library)
+
+4. Set up google auth
+
+#### Credentials
+
+Create credentials:
+  - type oAuth Client ID
+  - Application type: Other,
+  - Aplication Name: up to you, i.e. lcms-cli-dev
+
+Download client secret JSON file.
+
+###### Local development
+
+Copy downloaded JSON file to `config/google/client_secret.json`
+
+###### Server
+
+Make sure you're executing commands under _cloud66-user_
+```bash
+$ sudo -i -u cloud66-user
+```
+
+Make sure that the following directory exists on a server and is symlinked correctly
+```bash
+$ mkdir -p "$STACK_BASE/shared/google"
+$ chown cloud66-user:cloud66-user "$STACK_BASE/shared/google"
+$ ln -nsf "$STACK_BASE/shared/google" "$STACK_PATH/config/google"
+```
+
+Copy downloaded JSON file and set appropriate ownership and access rights for it (where `cloud66-user` is a system username under which web application is running)
+
+```bash
+$ cp client_secret.json $STACK_BASE/shared/google/
+$ cd $STACK_BASE/shared/google
+$ sudo chown cloud66-user:cloud66-user client_secret.json
+$ sudo chmod 755 client_secret.json
+```
+
+#### Application token
+
+If you generate token on a remote server, then make sure you're executing commands under _cloud66-user_
+
+```bash
+$ sudo -i -u cloud66-user
+```
+
+Run rake task. You will be asked to go by link, give app permissions and paste code into the terminal
+
+```bash
+$ bundle exec rake google:setup_auth
+```
+
+This will create `config/google/app_token.yaml`. Change the ownership of the `app_token.yaml` otherwise it will not be accessible for the application:
+
+```bash
+$ cd $STACK_BASE/shared/google
+$ chown cloud66-user:app_writers app_token.yaml
+$ sudo chmod g+w app_token.yaml
+```
+
+5. Add script
+- go to https://www.google.com/script/start/
+- copy-paste content of `config/scripts/Code.gs` there and save
+- at top menu Resources->Cloud Platform Project set project from step 2 (you need to paste *project number*)
+- at top menu Publish->Deploy As API Executable set version v1, access Only myself
+- save Current API ID somewhere, click Close on that annoying window (update will not close it)
+- choose any function and run it, it'll request permissions - grant them (there will be security warnings, just ignore them)
+
+6. Create folder (all materials will be saved there) on Google Drive, give view only to all by link, keep folder id (last part of url), copy to the root [LANDSCAPE](https://docs.google.com/document/d/1pXQDNKYOJYT6OTPnp8gsTWAydg5B9GTRibaWspmX4oE), [PORTRAIT](https://docs.google.com/document/d/1ijuZhGQXkPBxcZT4DRyNVY-qmI0xyVvSzVFqckOpsCc) templates and keep their IDs.
+
+7. Update corresponding env-file in the project
+
+```
+GOOGLE_APPLICATION_FOLDER_ID=saved from step 6
+GOOGLE_APPLICATION_SCRIPT_ID=saved id from step 5
+GOOGLE_APPLICATION_TEMPLATE_PORTRAIT=saved from step 6
+GOOGLE_APPLICATION_TEMPLATE_LANSCAPE=saved from step 6
+```
+
+## How to update Google Application Script
+
+1. Proceed to Google Drive for an account which has been used to create the Google Application Script (see _Add script_ section in [Set up](#set-up))
+
+2. Update script content
+
+3. Re-publish the script:
+
+- `Publish->Deploy As API Executable`
+- Set new version
+- `Update` & `Close`
+- Choose any function and run it, it'll request permissions - grant them (there will be security warnings, just ignore them)

data/lcms-engine.gemspec

@@ -23,77 +23,68 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'active_model_serializers', '~> 0.10.10'
   s.add_dependency 'activejob-retry', '~> 0.6.3'
-  s.add_dependency 'acts-as-taggable-on', '~> 6.5.0'
-  s.add_dependency 'acts_as_list', '~> 1.0.0'
-  s.add_dependency 'addressable', '~> 2.7.0'
-  s.add_dependency 'airbrake', '~> 10.0.1'
-  s.add_dependency 'autoprefixer-rails', '~> 9.7.4'
+  s.add_dependency 'acts-as-taggable-on', '~> 6.5'
+  s.add_dependency 'acts_as_list', '~> 1.0'
+  s.add_dependency 'addressable', '~> 2.7'
+  s.add_dependency 'airbrake', '~> 10.0'
+  s.add_dependency 'autoprefixer-rails', '~> 9.7'
   s.add_dependency 'aws-sdk-rails', '~> 3.1'
   s.add_dependency 'aws-sdk-s3', '~> 1'
-  s.add_dependency 'bitly', '~> 1.1.2'
-  s.add_dependency 'bootstrap-sass', '~> 3.4.1'
-  s.add_dependency 'bullet', '~> 6.1.0'
-  s.add_dependency 'carrierwave', '~> 2.1.0'
-  s.add_dependency 'ckeditor', '~> 4.3.0'
-  s.add_dependency 'closure_tree', '~> 7.1.0'
-  s.add_dependency 'combine_pdf', '~> 1.0.16'
-  s.add_dependency 'daemons', '~> 1.3.1'
-  s.add_dependency 'devise', '~> 4.7.1'
+  s.add_dependency 'bitly', '~> 1.1', '>= 1.1.2'
+  s.add_dependency 'bootstrap-sass', '~> 3.4'
+  s.add_dependency 'bullet', '~> 6.1', '>= 6.1.0'
+  s.add_dependency 'carrierwave', '~> 2.1'
+  s.add_dependency 'ckeditor', '~> 4.3'
+  s.add_dependency 'closure_tree', '~> 7.1'
+  s.add_dependency 'combine_pdf', '~> 1.0'
+  s.add_dependency 'daemons', '~> 1.3', '>= 1.3.1'
+  s.add_dependency 'devise', '~> 4.7', '>= 4.7.1'
   s.add_dependency 'elasticsearch-dsl', '~> 0.1.9'
-  s.add_dependency 'elasticsearch-model', '~> 7.0.0'
-  s.add_dependency 'elasticsearch-persistence', '~> 7.0.0'
-  s.add_dependency 'elasticsearch-rails', '~> 7.0.0'
-  s.add_dependency 'fog-aws', '~> 3.5.2'
-  s.add_dependency 'font-awesome-sass', '~> 5.12.0'
-  s.add_dependency 'foundation-rails', '~> 6.6.1.0'
+  s.add_dependency 'elasticsearch-model', '~> 7.0'
+  s.add_dependency 'elasticsearch-persistence', '~> 7.0'
+  s.add_dependency 'elasticsearch-rails', '~> 7.0'
+  s.add_dependency 'fog-aws', '~> 3.5', '>= 3.5.2'
+  s.add_dependency 'font-awesome-sass', '~> 5.12'
+  s.add_dependency 'foundation-rails', '~> 6.6', '>= 6.6.1'
   s.add_dependency 'google-api-client', '~> 0.37.1'
-  s.add_dependency 'heap', '~> 1'
   s.add_dependency 'hiredis', '~> 0.6.3'
-  s.add_dependency 'httparty', '~> 0.18.0'
-  s.add_dependency 'jbuilder', '~> 2.10.0'
-  s.add_dependency 'jquery-rails', '~> 4.3.5'
-  s.add_dependency 'js-routes', '~> 1.4.9'
+  s.add_dependency 'httparty', '~> 0.18'
+  s.add_dependency 'jbuilder', '~> 2.10'
+  s.add_dependency 'jquery-rails', '~> 4.3', '>= 4.3.5'
+  s.add_dependency 'js-routes', '~> 1.4', '>= 1.4.9'
   s.add_dependency 'lt-google-api', '~> 0.1.1'
-  s.add_dependency 'lt-lcms', '~> 0.2.0'
-  s.add_dependency 'migration_data', '~> 0.6.0'
-  s.add_dependency 'mini_magick', '~> 4.10.1'
+  s.add_dependency 'lt-lcms', '~> 0.3'
+  s.add_dependency 'migration_data', '~> 0.6'
+  s.add_dependency 'mini_magick', '~> 4.10', '>= 4.10.1'
   s.add_dependency 'nested_form', '~> 0.3.2'
   s.add_dependency 'nikkou', '~> 0.0.5'
-  s.add_dependency 'nokogiri', '~> 1.10.8'
-  s.add_dependency 'oj', '~> 3.10.2'
-  s.add_dependency 'oj_mimic_json', '~> 1.0.1'
-  s.add_dependency 'pandoc-ruby', '~> 2.0.2'
+  s.add_dependency 'nokogiri', '~> 1.10', '>= 1.10.8'
+  s.add_dependency 'oj', '~> 3.10', '>= 3.10.2'
+  s.add_dependency 'oj_mimic_json', '~> 1.0', '>= 1.0.1'
+  s.add_dependency 'pandoc-ruby', '~> 2.0', '>= 2.0.2'
   s.add_dependency 'pdfjs_viewer-rails', '~> 0.3.1'
-  s.add_dependency 'pg', '~> 1.2.2'
-  s.add_dependency 'pg_search', '~> 2.3.2'
-  s.add_dependency 'rack-mini-profiler', '~> 1.1.6'
-  s.add_dependency 'rails', '~> 5.2.4'
-  s.add_dependency 'rails-assets-classnames', '~> 2.2.5'
-  s.add_dependency 'rails-assets-es6-promise', '~> 4.2.4'
-  s.add_dependency 'rails-assets-eventemitter3', '~> 3.1.2'
-  s.add_dependency 'rails-assets-fetch', '~> 3.0.0'
-  s.add_dependency 'rails-assets-jstree', '~> 3.3.8'
-  s.add_dependency 'rails-assets-knockout', '~> 3.5.0'
-  s.add_dependency 'rails-assets-lodash', '~> 4.17.15'
-  s.add_dependency 'rails-assets-selectize', '~> 0.12.6'
-  s.add_dependency 'ransack', '~> 2.3.2'
-  s.add_dependency 'react-rails', '~> 2.6.1'
-  s.add_dependency 'redis', '~> 4.1.3'
-  s.add_dependency 'resque', '~> 2.0.0'
-  s.add_dependency 'resque-scheduler', '~> 4.4.0'
-  s.add_dependency 'rest-client', '~> 2.1.0'
-  s.add_dependency 'ruby-progressbar', '~> 1.10.1'
+  s.add_dependency 'pg', '~> 1.2', '>= 1.2.2'
+  s.add_dependency 'pg_search', '~> 2.3', '>= 2.3.2'
+  s.add_dependency 'rack-mini-profiler', '~> 1.1', '>= 1.1.6'
+  s.add_dependency 'rails', '~> 5.2', '>= 5.2.4'
+  s.add_dependency 'ransack', '~> 2.3', '>= 2.3.2'
+  s.add_dependency 'react-rails', '~> 2.6', '>= 2.6.1'
+  s.add_dependency 'redis', '~> 4.1', '>= 4.1.3'
+  s.add_dependency 'resque', '~> 2.0', '>= 2.0.0'
+  s.add_dependency 'resque-scheduler', '~> 4.4', '>= 4.4.0'
+  s.add_dependency 'rest-client', '~> 2.1', '>= 2.1.0'
+  s.add_dependency 'ruby-progressbar', '~> 1.10', '>= 1.10.1'
   s.add_dependency 'rubyzip', '~> 2'
-  s.add_dependency 'sanitize', '~> 5.1.0'
-  s.add_dependency 'sass-rails', '~> 5.1.0'
-  s.add_dependency 'simple_form', '~> 5.0.2'
+  s.add_dependency 'sanitize', '>= 5.1', '< 5.3'
+  s.add_dependency 'sass-rails', '~> 5.1', '>= 5.1.0'
+  s.add_dependency 'simple_form', '~> 5.0', '>= 5.0.2'
   s.add_dependency 'staccato', '~> 0.5.3'
   s.add_dependency 'truncate_html', '~> 0.9.3'
-  s.add_dependency 'turbolinks', '~> 5.2.1'
-  s.add_dependency 'validate_url', '~> 1.0.8'
-  s.add_dependency 'virtus', '~> 1.0.5'
-  s.add_dependency 'will_paginate', '~> 3.2.1'
-  s.add_dependency 'will_paginate-bootstrap', '~> 1.0.2'
+  s.add_dependency 'turbolinks', '~> 5.2', '>= 5.2.1'
+  s.add_dependency 'validate_url', '~> 1.0', '>= 1.0.8'
+  s.add_dependency 'virtus', '~> 1.0', '>= 1.0.5'
+  s.add_dependency 'will_paginate', '~> 3.2', '>= 3.2.1'
+  s.add_dependency 'will_paginate-bootstrap', '~> 1.0', '>= 1.0.2'
 
   s.add_development_dependency 'capybara', '~> 3.31'
   s.add_development_dependency 'database_cleaner-active_record', '~> 1.8'

data/lib/doc_template/document.rb

@@ -2,7 +2,7 @@
 
 module DocTemplate
   class Document
-    MAX_PARSE_ITERATIONS = 100
+    MAX_PARSE_ITERATIONS = 300
 
     # Contains the list of tags for which no parts should be created
     TAGS_WITHOUT_PARTS = [
@@ -178,7 +178,7 @@ module DocTemplate
 
       parts << {
         anchor: parsed_tag.anchor.to_s,
-        content: sanitized_content.squish,
+        content: parsed_tag.try(:without_squish?) ? sanitized_content : sanitized_content.squish,
         context_type: @opts[:context_type],
         data: parsed_tag.tag_data,
         materials: parsed_tag.materials,