rightmove_wrangler 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rightmove_wrangler.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Robert May
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# RightmoveWrangler
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rightmove_wrangler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rightmove_wrangler
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/rightmove_wrangler

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rightmove_wrangler'
+
+processor = RightmoveWrangler::Processor.new(ARGV)
+processor.run
+

data/bin/rightmove_wranglerd

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'daemons'
+
+Daemons.run "rightmove_wrangler"
+

data/lib/rightmove_wrangler.rb

@@ -0,0 +1,201 @@
+require "rubygems"
+require "rightmove_wrangler/version"
+require "fileutils"
+require "rightmove"
+require "optparse"
+require "faraday"
+require "addressable/uri"
+
+module RightmoveWrangler
+  class Processor
+    attr_accessor :directory_list
+    IMAGE_REGEX = /\.(jpg|jpeg|png|gif)/i
+
+    def initialize(args)
+      @args = args
+      @options = {}
+    end
+
+    def run
+      @opts = OptionParser.new(&method(:set_opts))
+      @opts.parse!(@args)
+      $stdout.puts @options.inspect
+      if @options.empty? || @options[:path].nil? || !File.directory?(@options[:path])
+        $stderr.puts 'Invalid path specified'
+        exit 1
+      else
+        loop do
+          begin
+            process!
+          rescue Exception => ex
+            raise ex if @options[:trace] || SystemExit === ex
+            $stderr.print "#{ex.class}: " if ex.class != RuntimeError
+            $stderr.puts ex.message
+          end
+          sleep 5
+        end
+      end
+    end
+
+    protected
+    def set_opts(opts)
+      opts.banner = "rightmove_wrangler is used to monitor a directory, parse RM format zip files, and submit them via post to an API"
+      
+      opts.on('-p', '--path PATH', 'Path to watch') do |path|
+        @options[:path] = path
+      end
+      
+      opts.on('-u', '--url URL', 'URL to post the data to') do |url|
+        @options[:url] = url
+      end
+
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+
+      opts.on_tail('-v', '--version', 'Print version') do
+        puts "rightmove_wrangler #{RightmoveWrangler::VERSION}"
+        exit
+      end
+    end
+    
+    private
+    def process!
+      files = Dir.entries(@options[:path])
+      if directory_list == files
+        $stdout.puts 'Nothing changed since last poll'
+        return
+      else
+        self.directory_list = files
+      end
+      
+      begin
+        threads = [] 
+        Dir.foreach(@options[:path]) do |file|
+          $stdout.puts "Checking #{file}"
+          if match = file =~ /\.(zip|blm)/i
+            threads << Thread.new do
+              $stdout.puts "Working on #{file}"
+              send("work_#{match[1]}_file".to_sym)
+            end
+          end
+        end
+        threads.each do |t|
+          t.join
+          $stdout.puts t[:output]
+        end
+      rescue Exception => ex
+        $stderr.puts ex.inspect
+        $stderr.puts ex.backtrace
+      end
+    end
+
+    def work_blm_file(file)
+      blm = BLM.new( File.open(file, "r").read )
+      rows = blm.data.collect do |row|
+        row_hash = {}
+        row.attributes.each do |key, value|
+          row_hash[key] = if value =~ IMAGE_REGEX
+            $stdout.puts "Instantiated file #{value}"
+            instantiate_from_dir(value, @options[:path])
+          else
+            value
+          end
+        end
+      end
+    end
+
+    def work_zip_file(file)
+      zip_file = Zip::ZipFile.open("#{@options[:path]}/#{file}")
+      archive = Rightmove::Archive.new(zip_file) 
+      
+      rows = archive.document.data.collect do |row|
+        row_hash = {}
+        # This is a bit odd, but essentially it actually turns media rows into files
+        row.attributes.each do |key, value|
+          row_hash[key] = if value =~ IMAGE_REGEX
+            $stdout.puts "Instantiated file #{value}"
+            instantiate_from_zip_file(value, zip_file)
+          else
+            value
+          end
+        end
+        row_hash
+      end
+
+      payload = {
+        row_set: {
+          tag: archive.branch_id,
+          timestamp: archive.timestamp.to_i,
+          rows: rows
+        }
+      }
+      post!(payload)
+    end
+
+    def instantiate_from_dir(file_name, dir)
+      path = File.join(file_name, dir)
+      if File.exists?(path)
+        $stdout.puts "Found file: #{path}"
+        instantiate_file File.open(path)
+      else
+        $stdout.puts "Couldn't find file: #{path}"
+        file_name
+      end
+    end
+
+    def instantiate_from_zip_file(file_name, zip_file)
+      matching_files = zip_file.entries.select {|v| v.to_s =~ /#{file_name}/ }
+      if matching_files.empty?
+        $stdout.puts "Couldn't find file: #{file_name}"
+        file_name
+      else
+        $stdout.puts "Found file: #{file_name}"
+        file = StringIO.new( zip_file.read(matching_files.first) )
+        instantiate_file(file, file_name)
+      end
+    end
+
+    def instantiate_file(file, file_name = nil, content_type = nil)
+      if !file.respond_to?(:original_filename)
+        file.class.class_eval { attr_accessor :original_filename }
+        file.original_filename = file_name
+      end
+
+      if !file.respond_to?(:content_type)
+        content_type ||= "image/jpg"
+        file.class.class_eval { attr_accessor :content_type }
+        file.content_type = content_type
+      end
+
+      Faraday::UploadIO.new(file, file_name, file.content_type)
+    end
+
+    def post!(payload)
+      if @options[:url].nil?
+        Thread.current[:output] = "Missing a URL to post the data to"
+      end
+      
+      uri = Addressable::URI.parse(@options[:url])    
+      conn = Faraday.new(url: uri.origin) do |builder|
+        builder.request  :multipart
+        builder.request  :url_encoded
+        builder.response :logger
+        builder.adapter  :net_http
+      end
+
+      params = uri.query_values.merge(payload)
+      response = conn.post uri.path, params
+      
+      if response.status == 200
+        Thread.current[:output] = "Server returned a successful response"
+        return true
+      else
+        Thread.current[:output] = "Non 200 response returned from API: #{response.inspect}"
+        return false
+      end
+    end
+  end
+  
+end

data/lib/rightmove_wrangler/version.rb

@@ -0,0 +1,3 @@
+module RightmoveWrangler
+  VERSION = "0.1.0"
+end

data/rightmove_wrangler.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/rightmove_wrangler/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Robert May"]
+  gem.email         = ["robotmay@gmail.com"]
+  gem.description   = %q{rightmove_wrangler is a command line utility for processing a directory of Rightmove .zip files and submitting them to an API.}
+  gem.summary       = %q{A command-line utility for processing a directory of Rightmove .zip files}
+  gem.homepage      = "http://github.com/corewebdesign/rightmove_wrangler"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "rightmove_wrangler"
+  gem.require_paths = ["lib"]
+  gem.version       = RightmoveWrangler::VERSION
+
+  gem.add_runtime_dependency 'rightmove'
+  gem.add_runtime_dependency 'blm'
+  gem.add_runtime_dependency 'faraday'
+  gem.add_runtime_dependency 'multipart-post'
+  gem.add_runtime_dependency 'addressable'
+  gem.add_runtime_dependency 'daemons'
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: rightmove_wrangler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Robert May
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rightmove
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: blm
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multipart-post
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: rightmove_wrangler is a command line utility for processing a directory
+  of Rightmove .zip files and submitting them to an API.
+email:
+- robotmay@gmail.com
+executables:
+- rightmove_wrangler
+- rightmove_wranglerd
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/rightmove_wrangler
+- bin/rightmove_wranglerd
+- lib/rightmove_wrangler.rb
+- lib/rightmove_wrangler/version.rb
+- rightmove_wrangler.gemspec
+homepage: http://github.com/corewebdesign/rightmove_wrangler
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A command-line utility for processing a directory of Rightmove .zip files
+test_files: []