capun 0.0.30 → 0.0.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4baa7ac853298bbe25092a0711f86178c2be77d6
-  data.tar.gz: 2a323905899516638368782a88698c7fed381eaf
+  metadata.gz: c9cad418ca5b3b3cbbcab859022a574e1f549892
+  data.tar.gz: 92424bad80df1c7957fc4085fd196db137bd5bae
 SHA512:
-  metadata.gz: bac8bab9b399d3b8f4cf1a8eb68c44c6f0e51637f77ec986791b48cc9c8c649d61fcf86134c265e2283011529428eba982e8b5ec067c19c943daa17f39d92b9b
-  data.tar.gz: 76fb48ee778f41e39115800837a073518006e46069d6efcb0fb198f2408f9645bce6f6515fde083ec5b7decf9cfbf75b018ef5f4f76c9eac4e504f06ff119b1a
+  metadata.gz: ce502e94d0245afc0d2de1a6e41223049b13a67bf6c53d50849cc8dacd5523363eb221dd8c34d8272c5af12ade0ad416dff745cdd825636b9a5ad945bc9e27cc
+  data.tar.gz: 25e78ac4a8917753230dedd420dabae1b96e7682a36c9e283d7a17f88b3a2d597e45c00483218a9962ecafc73eb3fb7d4b29c593f2c7d69511efd05fe7786d81

data/lib/capun/setup.rb

@@ -33,6 +33,10 @@ set :std_uploads, [
   # unicorn.config.rb
   {what: "config/deploy/unicorn.config.rb.erb", where: '#{shared_path}/config/unicorn.config.rb', upload: true, overwrite: true},
   # database.yml
+  {what: "config/deploy/backup.sh.erb", where: '#{shared_path}/backup.sh', upload: true, overwrite: true},
+  # backup.sh.erb
+  {what: "config/deploy/drivesink.py", where: '#{shared_path}/drivesink.py', upload: true, overwrite: true},
+  # backup.sh.erb
   {what: "config/deploy/database.yml.erb", where: '#{shared_path}/config/database.yml', upload: true, overwrite: true},
   # jenkins' config.xml
   {what: "config/deploy/jenkins.config.xml.erb", where: '/var/lib/jenkins/jobs/#{fetch(:application)}/config.xml', upload: -> { !!fetch(:addJenkins) }, overwrite: false},
@@ -49,6 +53,16 @@ set :std_symlinks, [
   {what: "application.yml", where: '#{release_path}/config/application.yml'},
   {what: "newrelic.yml", where: '#{release_path}/config/newrelic.yml'}
 ]
+namespace :backup do
+  desc 'Backup application'
+  task :exec do
+    on roles(:app) do
+      if fetch(:useBackups)
+          execute "sudo /home/#{fetch(:user)}/apps/#{fetch(:application)}/shared/backup.sh"
+      end
+    end
+  end
+end
 
 namespace :predeploy do
   namespace :install do
@@ -67,10 +81,10 @@ before 'deploy', 'predeploy:install:rvm_ruby'
 
 namespace :deploy do
 
-  desc 'Kills running processes'
+  desc 'Kills unicorn processes'
   task :kill_me do
     on roles(:app) do
-      execute "kill -9 $(ps aux | grep #{fetch(:application)} | grep -v grep | awk '{print $2}') || true"
+      execute "cd /home/#{fetch(:user)}/apps/#{fetch(:application)}/shared/tmp/pids; for line in $(ls | grep unicorn); do  kill -15 $(sudo cat $line) || true ; done;"
     end
   end
   before :deploy, 'deploy:kill_me'
@@ -175,6 +189,18 @@ namespace :deploy do
     end
   end
 
+  desc 'Update cron backup task'
+  task :update_cron do
+    if fetch(:useBackups)
+      on roles(:app) do
+        execute :chmod, "+x #{shared_path}/backup.sh"
+        info "making backup.sh executable"
+        execute :sudo, :ln, "-nfs", "#{shared_path}/backup.sh /etc/cron.#{fetch(:backupTime)}/backup-#{fetch(:application).gsub(/\./, '-')}"
+        info "Create symbolic link for backup"
+      end
+    end
+  end
+
 end
 
 before "deploy:updating", "deploy:make_dirs"
@@ -185,4 +211,5 @@ after "deploy:publishing", "deploy:set_up_jenkins"
 after "deploy:publishing", "deploy:prepare_logrotate"
 after "deploy:publishing", "deploy:restart_nginx"
 after "deploy:publishing", "deploy:restart_logstash"
+after "deploy:publishing", "deploy:update_cron"
 after "deploy:publishing", "unicorn:legacy_restart"

data/lib/capun/version.rb

@@ -1,3 +1,3 @@
 module Capun
-  VERSION = "0.0.30"
+  VERSION = "0.0.31"
 end

data/lib/generators/capun/stage_generator.rb

@@ -22,12 +22,14 @@ module Capun
         end
         @addELK = ask("Would you like to add ELK-compatible logging? [Y/n]").capitalize == 'Y'
         @addlogrotate = ask("Would you like to add logrotate configuration to stage? [Y/n]").capitalize == 'Y'
+        @useBackups = ask("Would you like to add amazon backup system? [Y/n]").capitalize == 'Y'
       end
 
       def add_stage
         template "stage.rb.erb", "config/deploy/#{singular_name}.rb"
       end
 
+
       def copy_env_file
         copy_file Rails.root.join('config', 'environments', 'production.rb'), "config/environments/#{singular_name}.rb"
       end
@@ -85,6 +87,19 @@ module Capun
         end
       end
 
+      def useBackups
+        if @useBackups
+          append_to_file "config/deploy/#{singular_name}.rb", "#backup_system\n" +
+          "set :useBackups, true\n" +
+          "set :backupTime, \"daily\" # available hourly, daily, monthly, weekly\n" +
+          "set :backupFolders, %w{public/system} #recursive\n" +
+          "#set :slack_hook, [hook]\n" +
+          "#set :slack_channel, [channel] #must be specified"
+          copy_file "backup.sh.erb", "config/deploy/backup.sh.erb"
+          copy_file "drivesink.py", "config/deploy/drivesink.py"
+        end
+      end
+
       def add_jenkins
         if @addJenkins
           copy_file "jenkins.config.xml.erb", "config/deploy/jenkins.config.xml.erb"
@@ -105,4 +120,4 @@ module Capun
       end
     end
   end
-end
+end

data/lib/generators/capun/templates/backup.sh.erb

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+cd /home/<%= fetch(:user) %>/apps/<%= fetch(:application) %>/shared
+curdate=$(date +"%m-%d-%y")
+mkdir -p "backuping/version-$curdate"
+
+zip -u -0 -r -s 1000 "backuping/version-$curdate/assets.zip" <% fetch(:backupFolders).each do |folder|%> <%= folder + ' ' %> <% end %>
+
+adapter=$(cat config/database.yml | grep adapter | sed s/adapter://g | xargs)
+if [ "$adapter" = "sqlite3" ]; then
+  DATA_BASE_TYPE="SQLite3"
+  cp "<%=fetch(:stage)%>.sqlite3" "backuping/version-$curdate/database.sqlite3"
+fi
+if [ "$adapter" = "mysql2" ]; then
+DATA_BASE_TYPE="MySQL"
+username=$(cat config/database.yml | grep username | sed s/username://g | xargs)
+password=$(cat config/database.yml | grep password | sed s/password://g | xargs)
+database=$(cat config/database.yml | grep database | sed s/database://g | xargs)
+mysqldump --user="$username" --password="$password" $database > "backuping/version-$curdate/database.dump"
+fi
+
+python drivesink.py upload "backuping" <%= fetch(:backupDestinationFolder) || "backups/firstdedic-server/" + fetch(:application)  %>
+
+if [ $? -eq 0 ]; then
+SIZE=$(du -sh backuping | awk '{print $1}')
+curl -X POST -H 'Content-type: application/json' \
+--data "{'attachments': [{'mrkdwn_in': ['text'],'text': 'Успешо создан бэкап для *<%= fetch(:application) %>*\n<% fetch(:backupFolders).each do |folder|%><%= "-" +folder + '\n' %><% end %>Тип базы данных: *$DATA_BASE_TYPE*\nРазмер *$SIZE*','color': '#3AA3E3', 'title': 'Amazon Backup System'}]<%=", \'channel\': \'#{fetch(:slack_channel)}\' " if !fetch(:slack_channel).nil? %>}" \
+<%= fetch(:slack_hook) %>
+else
+curl -X POST -H 'Content-type: application/json' \
+--data "{'attachments': [{'mrkdwn_in': ['text'],'text': 'Произошла ошибка при создании бэкапа для *<%= fetch(:application) %>','color': '#FF0000', 'title': 'Amazon Backup System'}]<%=", \'channel\': \'#{fetch(:slack_channel)}\' " if !fetch(:slack_channel).nil? %>}" \
+<%= fetch(:slack_hook) %>
+fi
+
+rm -rf backuping

data/lib/generators/capun/templates/drivesink.py

@@ -0,0 +1,336 @@
+#!/usr/bin/env python
+#--*-- coding: utf-8 --*--
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import requests
+import requests_toolbelt
+import sys
+import uuid
+import mimetypes
+import inspect
+
+class CloudNode(object):
+    def __init__(self, node):
+        self.node = node
+        self._children_fetched = False
+
+    def children(self):
+        if not self._children_fetched:
+            nodes = DriveSink.instance().request_metadata(
+                "%%snodes/%s/children" % self.node["id"])
+            self._children = {n["name"]: CloudNode(n) for n in nodes["data"]}
+            self._children_fetched = True
+        return self._children
+
+    def child(self, name, create=False):
+        node = self.children().get(name)
+        if not node and create:
+            node = self._make_child_folder(name)
+        return node
+
+    def upload_child_file(self, name, local_path, existing_node=None):
+        logging.info("Uploading %s to %s", local_path, self.node["name"])
+        mime_type = _get_mimetype(name)
+        m = requests_toolbelt.MultipartEncoder([
+            ("metadata", json.dumps({
+                "name": name, # Здесь можно задать имя файлу, которое будет у него в облаке
+                "kind": "FILE",
+                "parents": [self.node["id"]]
+            })),
+            ("content", (name, open(local_path, "rb"), mime_type))])
+        if existing_node:
+            """
+            # TODO: this is under-documented and currently 500s on Amazon's side
+            node = CloudNode(DriveSink.instance().request_content(
+                "%%snodes/%s/content" % existing_node.node["id"],
+                method="put", data=m, headers={"Content-Type": m.content_type}))
+            """
+            old_info = DriveSink.instance().request_metadata(
+                "%%s/trash/%s" % existing_node.node["id"], method="put")
+        node = CloudNode(DriveSink.instance().request_content(
+            "%snodes", method="post", data=m,
+            headers={"Content-Type": m.content_type}))
+        self._children[name] = node
+
+    def download_file(self, local_path):
+        logging.info("Downloading %s into %s", self.node["name"], local_path)
+        req = DriveSink.instance().request_content(
+            "%%snodes/%s/content" % self.node["id"], method="get", stream=True,
+            decode=False)
+        if req.status_code != 200:
+            logging.error("Unable to download file: %r", req.text)
+            sys.exit(1)
+        with open(local_path, "wb") as f:
+            for chunk in req.iter_content():
+                f.write(chunk)
+
+    def differs(self, local_path):
+
+        # Этот закомментированный блок проверяет загружаемые файлы на дубликатность... как-то так
+        #
+        #return (not os.path.exists(local_path) or
+        #        self.node["contentProperties"]["size"] !=
+        #        os.path.getsize(local_path) or
+        #        self.node["contentProperties"]["md5"] !=
+        #        self._md5sum(local_path))
+
+        return True
+
+
+    def _md5sum(self, filename, blocksize=65536):
+        md5 = hashlib.md5()
+        with open(filename, "rb") as f:
+            for block in iter(lambda: f.read(blocksize), ""):
+                md5.update(block)
+        return md5.hexdigest()
+
+    def _make_child_folder(self, name):
+        logging.info(
+            "Creating remote folder %s in %s", name, self.node["name"])
+        node = CloudNode(
+            DriveSink.instance().request_metadata("%snodes", {
+                "kind": "FOLDER",
+                "name": name,
+                "parents": [self.node["id"]]}))
+        self._children[name] = node
+        return node
+
+
+class DriveSink(object):
+    def __init__(self, args):
+        if not args:
+            logging.error("Never initialized")
+            sys.exit(1)
+        self.args = args
+        self.config = None
+
+    @classmethod
+    def instance(cls, args=None):
+        if not hasattr(cls, "_instance"):
+            cls._instance = cls(args)
+        return cls._instance
+
+    def upload(self, source, destination):
+        remote_node = self.node_at_path(
+            self.get_root(), destination, create_missing=True)
+        for dirpath, dirnames, filenames in os.walk(source):
+            relative = dirpath[len(source):]
+            current_dir = self.node_at_path(
+                remote_node, relative, create_missing=True)
+            if not current_dir:
+                logging.error("Could not create missing node")
+                sys.exit(1)
+            for dirname in dirnames:
+                logging.info(dirnames)
+                current_dir.child(dirname, create=True)
+            for filename in filenames:
+                local_path = os.path.join(dirpath, filename)
+                node = current_dir.child(filename)
+                if (not node or node.differs(
+                        local_path)) and self.filter_file(filename):
+                    current_dir.upload_child_file(filename, local_path, node)
+
+    def upload_file(self, source, destination):
+        '''
+            funkcija poluchaet polnyj put' k fajlu, soderzhashhij imja i rasshirenie fajla (primer: /home/ubuntu/upl/index.html)
+            i razbivaet ego na put' k fajlu (dirpath=/home/ubuntu/upl/) i nazvanie fajla (filename=index.html)
+
+            zatem dejstvuet po analogii s funkciej "upload"
+        '''
+        remote_node = self.node_at_path(
+            self.get_root(), destination, create_missing=True)
+
+        dirpath = os.path.split(source)[0]
+        filename = os.path.split(source)[1]
+
+        relative = dirpath[len(dirpath):]
+        current_dir = self.node_at_path(
+            remote_node, relative, create_missing=True)
+
+        local_path = os.path.join(dirpath, filename)
+        node = current_dir.child(filename)
+        current_dir.upload_child_file(filename, local_path, node)
+
+    def download(self, source, destination):
+        to_download = [(self.node_at_path(self.get_root(), source),
+                        self.join_path(destination, create_missing=True))]
+        while len(to_download):
+            node, path = to_download.pop(0)
+            for name, child in node.children().iteritems():
+                if child.node["kind"] == "FOLDER":
+                    to_download.append((child, self.join_path(
+                        child.node["name"], path, create_missing=True)))
+                elif child.node["kind"] == "FILE":
+                    local_path = os.path.join(path, child.node["name"])
+                    if child.differs(local_path):
+                        child.download_file(local_path)
+
+    def filter_file(self, filename):
+        _, extension = os.path.splitext(filename)
+        extension = extension.lstrip(".").lower()
+
+    # Этот закомментированный блок проверяет расширения файлов и если они есть в Allowed, то он их загружает в облако.
+    # Если нужна фильтрация файлов, то раскомментируйте этот блок, а строку "return extension" - закомментируйте
+    #
+    #    allowed = self.args.extensions
+    #    if not allowed:
+    #        # Not all tested to be free
+    #        allowed = (
+    #            "apng,arw,bmp,cr2,crw,dng,emf,gif,jfif,jpe,jpeg,jpg,mef,nef,"
+    #            "orf,pcx,png,psd,raf,ras,srw,swf,tga,tif,tiff,wmf")
+    #    return extension in allowed.split(",")
+
+        return extension
+
+    def get_root(self):
+        nodes = self.request_metadata("%snodes?filters=isRoot:true")
+        if nodes["count"] != 1:
+            logging.error("Could not find root")
+            sys.exit(1)
+        return CloudNode(nodes["data"][0])
+
+    def node_at_path(self, root, path, create_missing=False):
+        parts = filter(None, path.split("/"))
+        node = root
+        while len(parts):
+            node = node.child(parts.pop(0), create=create_missing)
+            if not node:
+                return None
+        return node
+
+    def join_path(self, destination, root="/", create_missing=True):
+        directory = os.path.join(root, destination)
+        if not os.path.exists(directory):
+            if create_missing:
+                os.makedirs(directory)
+            else:
+                return None
+        if not os.path.isdir(directory):
+            logging.error("%s is not a directory", directory)
+            sys.exit(1)
+        return directory
+
+    def _config_file(self):
+        config_filename = self.args.config or os.environ.get(
+            "DRIVESINK", None)
+        if not config_filename:
+            config_filename = os.path.join(
+                os.path.expanduser("~"), ".drivesink")
+        return config_filename
+
+    def _config(self):
+        if not self.config:
+            config_filename = self._config_file()
+            try:
+                self.config = json.loads(open(config_filename, "r").read())
+            except:
+                print "%s/config to get your tokens" % self.args.drivesink
+                sys.exit(1)
+        return self.config
+
+    def request_metadata(self, path, json_data=None, **kwargs):
+        args = {}
+        if json_data:
+            args["method"] = "post"
+            args["data"] = json.dumps(json_data)
+        else:
+            args["method"] = "get"
+
+        args.update(kwargs)
+
+        return self._request(
+            path % self._config()["metadataUrl"], **args)
+
+    def request_content(self, path, **kwargs):
+        # приписка '?suppress=deduplication' дает возможность добавлять одинаковые файлы в разные папки.
+        # если имена файлов одинаковы, то они замещаются при этом хэш-сумма не учитывается.
+        return self._request(
+            (path + '?suppress=deduplication') % self._config()["contentUrl"], **kwargs)
+
+    def _request(self, url, refresh=True, decode=True, **kwargs):
+        headers = {
+            "Authorization": "Bearer %s" % self._config()["access_token"],
+        }
+        headers.update(kwargs.pop("headers", {}))
+        req = requests.request(url=url, headers=headers, **kwargs)
+
+        if req.status_code == 401 and refresh:
+            # Have to proxy to get the client id and secret
+            req = requests.post("%s/refresh" % self.args.drivesink, data={
+                "refresh_token": self._config()["refresh_token"],
+            })
+            if req.status_code != 200:
+                try:
+                    response = req.json()
+                    logging.error("Got Amazon code %s: %s",
+                                  response["code"], response["message"])
+                    sys.exit(1)
+                except Exception:
+                    pass
+            req.raise_for_status()
+            try:
+                new_config = req.json()
+            except:
+                logging.error("Could not refresh: %r", req.text)
+                raise
+            self.config.update(new_config)
+            with open(self._config_file(), "w") as f:
+                f.write(json.dumps(self.config, sort_keys=True, indent=4))
+            return self._request(url, refresh=False, decode=decode, **kwargs)
+        if req.status_code != 200:
+            try:
+                response = req.json()
+                logging.error("Got Amazon code %s: %s",
+                              response["code"], response["message"])
+                sys.exit(1)
+            except Exception:
+                pass
+        # Здесь можно организовать уведомление об успешной загрузке - код ответа 200
+        if req.status_code == 200:
+            logging.info('Отправка данных: <' + inspect.stack()[1][3] + '> Код ответа: 200. ОК.')
+
+        req.raise_for_status()
+        if decode:
+            return req.json()
+        return req
+
+def _get_mimetype(file_name = ''):
+    mt = mimetypes.guess_type(file_name)[0]
+    return mt if mt else 'application/octet-stream'
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Amazon Cloud Drive synchronization tool")
+    parser.add_argument("command", choices=["upload", "download"],
+                        help="Commands: 'upload' or 'download'")
+    parser.add_argument("source", help="The source directory")
+    parser.add_argument("destination", help="The destination directory")
+    parser.add_argument("-e", "--extensions",
+                        help="File extensions to upload, images by default")
+    parser.add_argument("-c", "--config", help="The config file")
+    parser.add_argument("-d", "--drivesink", help="Drivesink URL",
+                        default="https://drivesink.appspot.com")
+    args = parser.parse_args()
+
+    drivesink = DriveSink.instance(args)
+
+    if args.command == "upload":
+        if(not os.path.isfile(args.source)):
+            drivesink.upload(args.source, args.destination)
+        else:
+            drivesink.upload_file(args.source, args.destination)
+    elif args.command == "download":
+        drivesink.download(args.source, args.destination)
+
+logging.basicConfig(
+    format = "%(levelname) -10s %(module)s:%(lineno)s %(funcName)s %(message)s",
+    level = logging.DEBUG
+)
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+if __name__ == "__main__":
+    main()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: capun
 version: !ruby/object:Gem::Version
-  version: 0.0.30
+  version: 0.0.31
 platform: ruby
 authors:
 - Ivan Zamylin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -129,9 +129,11 @@ files:
 - lib/generators/capun/stage_generator.rb
 - lib/generators/capun/templates/Capfile
 - lib/generators/capun/templates/append_info.excerpt
+- lib/generators/capun/templates/backup.sh.erb
 - lib/generators/capun/templates/basic_authenticatable.rb.erb
 - lib/generators/capun/templates/database.yml.erb
 - lib/generators/capun/templates/deploy.rb.erb
+- lib/generators/capun/templates/drivesink.py
 - lib/generators/capun/templates/jenkins.config.xml.erb
 - lib/generators/capun/templates/lograge_env_config.excerpt
 - lib/generators/capun/templates/lograge_initializer.rb