filer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 94642ecef0095a70c043f4b1b213c4c47b952c8f
+  data.tar.gz: c395f12f7f090814062b3a0b467a0e6498b4d5a5
+SHA512:
+  metadata.gz: 9cc75cdbfdc451d8e70520a26fac561b6cdf3b86d28eea490796f21a1400b79a99abff66059e63ba073f89aa2e9eaccfa83d0a5caa3d0787b895cf2f9dfe209c
+  data.tar.gz: 89d649680dbcd2a079775c1673a1bdb3806afba4f038f22f78f132e3104d4c30f570164c69c4957eb17f2362cbcefe4c28ee8ba5ec3698df1768f4f9df435e54

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.log
+*.pid

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in filer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Brian Zeligson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Filer
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'filer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install filer
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/filer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/filer

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'filer'
+
+Filer::Command.start(ARGV)

data/filer.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'filer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "filer"
+  spec.version       = Filer::VERSION
+  spec.authors       = ["Brian Zeligson"]
+  spec.email         = ["brian.zeligson@gmail.com"]
+  spec.summary       = %q{Indexes files and stores to s3}
+  spec.description   = %q{Indexes files and stores to s3}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "thor"
+  spec.add_development_dependency "configliere"
+  spec.add_development_dependency "notifier"
+  spec.add_development_dependency "aws-sdk"
+  spec.add_development_dependency "flex"
+  spec.add_development_dependency "flex-models"
+  spec.add_development_dependency "fallen"
+end

data/lib/filer.rb

@@ -0,0 +1,91 @@
+require "filer/version"
+require 'filer/watcher'
+require "yaml"
+require "thor"
+require "configliere"
+
+module Filer
+  CONFIG = '.filer.yml'
+
+  class Command < Thor
+
+    no_commands do
+      def watcher
+        Watcher.instance(Settings)
+      end
+      
+      def save_settings
+        Settings.save!(CONFIG)
+      end
+      
+      def s3
+        params = [:s3_key, :s3_secret, :s3_bucket].map {|m| Settings[m]}
+        unless params.compact.size == 3
+          puts "Please run filer configure-s3 first"
+          exit
+        end
+        @s3 ||= Filer::S3.new(*params)
+      end
+    end
+
+    desc "start-watching", "start watching configured directories"
+    def start_watching
+      watcher.daemonize!
+      watcher.start!
+    end
+
+    desc "stop-watching", "stop watching configured directories"
+    def stop_watching
+      watcher.stop!
+    end
+
+    desc "directories", "List directories currently handled by filer"
+    def directories
+      Settings[:directories].each_with_index do |dir, i|
+        puts "#{i+1}) #{dir}"
+      end
+    end
+
+    desc "add-directory", "Adds a directory to be handled by filer"
+    def add_directory(dir)
+      Settings[:directories].push(dir)
+      save_settings
+    end
+
+    desc "remove-directory", "Removes directory with given index. Index is taken from " <<
+                           "output of filer directories"
+    def remove_directory(i)
+      Settings[:directories].delete_at(i.to_i-1)
+      save_settings
+      puts "Updated. New directories: "
+      directories
+    end
+
+    desc "configure-s3 KEY SECRET BUCKET", 
+      "Define key, secret, and bucket to use for s3 " <<
+      "upload, eg: filer configure-s3 my_key my_secret my_bucket"
+    def configure_s3(key, secret, bucket)
+      Settings[:s3_key], Settings[:s3_secret], Settings[:s3_bucket] =
+        key, secret, bucket
+      save_settings
+    end
+
+    desc 'search "KEYWORDS"', "Search indexed files"
+    def search(keywords)
+      results = Filer::Filed.searchable(keywords).all
+      if results.empty?
+        puts "No results found"
+        return
+      end
+      results.each_with_index do |r, i|
+        puts "#{i+1}) #{r.key} - #{r.highlight["attachment"].join(" ... ")}" 
+      end
+      ix = ask("Index of file to open (q to exit):")
+      return unless ix.to_i > 0 && results[ix.to_i - 1]
+      s3.open_file(results[ix.to_i - 1].key.first) 
+    end
+
+    Settings.read(CONFIG)
+    Settings[:directories] ||= []
+  end
+end

data/lib/filer/filed.rb

@@ -0,0 +1,38 @@
+require 'flex'
+require 'flex-models'
+
+module Filer
+  class Filed
+    include Flex::ActiveModel
+
+    INDEX = 'filer_filed'
+
+    def self.ensure_index
+      return if Flex.exist?(index: INDEX)
+      params = self.flex.default_mapping.clone[INDEX]
+      params[:settings] = { number_of_replicas: 1,
+                            number_of_shards: 4 }
+                            puts params
+      Flex.POST("/#{INDEX}", params)
+    end
+    
+    flex.index = INDEX
+
+    attribute :key
+    attribute_timestamps
+    attribute_attachment
+
+    def flex_id
+      key.gsub("/", "_")
+    end
+
+    scope :searchable do |q|
+       attachment_scope
+      .highlight(:fields => { :attachment          => {},
+                              :'attachment.title'  => {} },
+                 :pre_tags => ["*"], :post_tags => ["*"])
+      .query_string(q)
+    end 
+  end
+end
+Filer::Filed.ensure_index

data/lib/filer/s3.rb

@@ -0,0 +1,57 @@
+require 'aws-sdk'
+require 'tempfile'
+
+module Filer
+  class S3
+    
+    def initialize(key, secret, bucket_name)
+      @key, @secret, @bucket_name = key, secret, bucket_name
+    end
+
+    def s3_init_params
+      { access_key_id: @key, secret_access_key: @secret,
+        server_side_encryption: :aes256 }
+    end
+
+    def s3
+      @s3 ||= AWS::S3.new(s3_init_params)
+    end
+
+    def bucket
+      return @bucket if @bucket 
+      bucket = s3.buckets[@bucket_name]
+      unless bucket.location_constraint == s3.config.region
+        @s3 = nil
+        @s3 = AWS::S3.new(s3_init_params.
+          merge(region: bucket.location_constraint))
+        bucket = s3.buckets[@bucket_name]
+      end
+      @bucket = bucket
+    end
+
+    def s3_key(path)
+      filename = File.basename(path)
+      y = Time.now.strftime("%Y")
+      m = Time.now.strftime("%m")
+      d = Time.now.strftime("%d")
+      "#{y}/#{m}/#{d}/#{filename}"
+    end
+
+    def put_file(path)
+      o = bucket.objects[s3_key(path)]
+      o.write(Pathname.new(path), server_side_encryption: :aes256)
+    end
+
+    def open_file(key)
+      o = bucket.objects[key]
+      filename = File.basename(key)
+      ext = File.extname(key)
+      t = Tempfile.new([filename, ext])
+      o.read do |ch| t.write(ch) end
+      t.close
+      # seems like a race condition
+      sleep 1
+      `open #{t.path}`
+    end
+  end
+end

data/lib/filer/version.rb

@@ -0,0 +1,3 @@
+module Filer
+  VERSION = "0.0.1"
+end

data/lib/filer/watcher.rb

@@ -0,0 +1,53 @@
+require "fallen"
+require "notifier"
+require "filer/s3"
+require "filer/filed"
+
+module Watcher
+  extend Fallen
+  FILER_LOG = "filer_watcher.log"
+
+  def self.instance(settings)
+    @settings = settings
+    Watcher.pid_file 'filer_watcher.pid'
+    Watcher.stderr FILER_LOG
+    self.s3
+    self
+  end
+
+  def self.s3
+    params = [:s3_key, :s3_secret, :s3_bucket].map {|m| @settings[m]}
+    unless params.compact.size == 3
+      puts "Please run filer configure-s3 first"
+      exit
+    end
+    @s3 ||= Filer::S3.new(*params)
+  end
+
+  def self.notify(msg)
+    Notifier.notify(
+      title: "Filer",
+      message: msg)
+  end
+
+  def self.run
+    while running?
+      begin
+        files = @settings[:directories].flat_map {|d| Dir["#{d}/*"]}
+        files.each do |f|
+          @s3.put_file(f)
+          fd = Filer::Filed.new(
+            key: @s3.s3_key(f), 
+            attachment: Base64.encode64(File.read(f)))
+          fd.save
+          File.delete(f)
+        end
+        self.notify("Processed #{files.size} files") unless files.empty?
+        sleep 20
+      rescue Exception => e
+        self.notify("Filer encountered an error. " <<
+          "See #{File.dirname(__FILE__)}/#{FILER_LOG}")
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: filer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brian Zeligson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: configliere
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: notifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flex-models
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fallen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Indexes files and stores to s3
+email:
+- brian.zeligson@gmail.com
+executables:
+- filer
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/filer
+- filer.gemspec
+- lib/filer.rb
+- lib/filer/filed.rb
+- lib/filer/s3.rb
+- lib/filer/version.rb
+- lib/filer/watcher.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Indexes files and stores to s3
+test_files: []
+has_rdoc: