rivendell-import 0.0.4 → 0.0.5

data/Gemfile

@@ -1,11 +1,10 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gemspec
 
-gem "rivendell-api" # , :git => "git://projects.tryphon.priv/rivendell-api", :ref => "b98a46fbb0c31d965853206a56500cea4a340044" # , :path => "~/Projects/RivendellApi"
+# gem "rivendell-db", :path => "~/Projects/RivendellDb"
 
 if RUBY_PLATFORM =~ /linux/
-  gem 'libnotify' 
-  gem 'rb-inotify', '~> 0.8.8'
+  gem 'libnotify'
+  gem 'rb-inotify'
 end
-

data/Gemfile.lock

@@ -1,36 +1,39 @@
 PATH
   remote: .
   specs:
-    rivendell-import (0.0.4)
+    rivendell-import (0.0.5)
       SyslogLogger (~> 2.0)
       activerecord (~> 3.2.8)
       activesupport (~> 3.2.8)
       daemons
       httmultiparty
-      listen
+      listen (~> 1.2.1)
       mail
       rivendell-api (~> 0.0.4)
+      rivendell-db
       sinatra
       sqlite3
       trollop
       will_paginate (~> 3.0.0)
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
     SyslogLogger (2.0)
-    activemodel (3.2.9)
-      activesupport (= 3.2.9)
+    activemodel (3.2.13)
+      activesupport (= 3.2.13)
       builder (~> 3.0.0)
-    activerecord (3.2.9)
-      activemodel (= 3.2.9)
-      activesupport (= 3.2.9)
+    activerecord (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
       arel (~> 3.0.2)
       tzinfo (~> 0.3.29)
-    activesupport (3.2.9)
-      i18n (~> 0.6)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
       multi_json (~> 1.0)
+    addressable (2.3.5)
     arel (3.0.2)
+    bcrypt-ruby (3.0.1)
     builder (3.0.3)
     cucumber (1.2.1)
       builder (>= 2.1.2)
@@ -38,9 +41,38 @@ GEM
       gherkin (~> 2.11.0)
       json (>= 1.4.6)
     daemons (1.1.9)
+    data_objects (0.10.13)
+      addressable (~> 2.1)
     database_cleaner (0.8.0)
     diff-lcs (1.1.3)
-    ffi (1.1.5)
+    dm-core (1.2.1)
+      addressable (~> 2.3)
+    dm-do-adapter (1.2.0)
+      data_objects (~> 0.10.6)
+      dm-core (~> 1.2.0)
+    dm-mysql-adapter (1.2.0)
+      dm-do-adapter (~> 1.2.0)
+      do_mysql (~> 0.10.6)
+    dm-serializer (1.2.2)
+      dm-core (~> 1.2.0)
+      fastercsv (~> 1.5)
+      json (~> 1.6)
+      json_pure (~> 1.6)
+      multi_json (~> 1.0)
+    dm-types (1.2.2)
+      bcrypt-ruby (~> 3.0)
+      dm-core (~> 1.2.0)
+      fastercsv (~> 1.5)
+      json (~> 1.6)
+      multi_json (~> 1.0)
+      stringex (~> 1.4)
+      uuidtools (~> 2.1)
+    dm-validations (1.2.0)
+      dm-core (~> 1.2.0)
+    do_mysql (0.10.13)
+      data_objects (= 0.10.13)
+    fastercsv (1.5.5)
+    ffi (1.9.0)
     gherkin (2.11.2)
       json (>= 1.4.6)
     guard (1.4.0)
@@ -52,37 +84,49 @@ GEM
     guard-rspec (2.0.0)
       guard (>= 1.1)
       rspec (~> 2.11)
-    httmultiparty (0.3.8)
+    httmultiparty (0.3.10)
       httparty (>= 0.7.3)
       multipart-post
-    httparty (0.9.0)
+    httparty (0.11.0)
       multi_json (~> 1.0)
-      multi_xml
+      multi_xml (>= 0.5.2)
     i18n (0.6.1)
     json (1.7.5)
+    json_pure (1.8.0)
     libnotify (0.8.0)
       ffi (>= 1.0.11)
-    listen (0.5.3)
-    mail (2.5.2)
-      i18n (>= 0.4.0)
+    listen (1.2.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+      rb-kqueue (>= 0.2)
+    mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.19)
+    mime-types (1.23)
     multi_json (1.4.0)
-    multi_xml (0.5.1)
-    multipart-post (1.1.5)
+    multi_xml (0.5.4)
+    multipart-post (1.2.0)
     null_logger (0.0.1)
     polyglot (0.3.3)
-    rack (1.4.1)
-    rack-protection (1.2.0)
+    rack (1.5.2)
+    rack-protection (1.5.0)
       rack
     rake (0.9.2.2)
-    rb-inotify (0.8.8)
+    rb-fsevent (0.9.3)
+    rb-inotify (0.9.0)
+      ffi (>= 0.5.0)
+    rb-kqueue (0.2.0)
       ffi (>= 0.5.0)
     rivendell-api (0.0.4)
       activesupport
       httmultiparty
       null_logger
+    rivendell-db (0.0.1)
+      dm-core
+      dm-mysql-adapter
+      dm-serializer
+      dm-types
+      dm-validations
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
       rspec-expectations (~> 2.11.0)
@@ -95,19 +139,21 @@ GEM
       multi_json (~> 1.0)
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
-    sinatra (1.3.3)
-      rack (~> 1.3, >= 1.3.6)
-      rack-protection (~> 1.2)
-      tilt (~> 1.3, >= 1.3.3)
-    sqlite3 (1.3.6)
+    sinatra (1.4.3)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    sqlite3 (1.3.7)
+    stringex (1.5.1)
     thor (0.16.0)
-    tilt (1.3.3)
-    treetop (1.4.12)
+    tilt (1.4.1)
+    treetop (1.4.14)
       polyglot
       polyglot (>= 0.3.1)
     trollop (2.0)
-    tzinfo (0.3.35)
-    will_paginate (3.0.3)
+    tzinfo (0.3.37)
+    uuidtools (2.1.4)
+    will_paginate (3.0.4)
 
 PLATFORMS
   ruby
@@ -120,8 +166,7 @@ DEPENDENCIES
   guard-rspec
   libnotify
   rake
-  rb-inotify (~> 0.8.8)
-  rivendell-api
+  rb-inotify
   rivendell-import!
   rspec
   simplecov

data/examples/config.rb

@@ -15,7 +15,6 @@ end
 Rivendell::Import::Notifier::Mail.from = "root@tryphon.eu"
 
 Rivendell::Import.config do |config|
-
   config.rivendell.host = "localhost"
   config.rivendell.login_name = "user"
   config.rivendell.password = ""
@@ -29,7 +28,7 @@ Rivendell::Import.config do |config|
       cart.clear_cuts!
       cart.find_by_title file.basename
     end
-    
+
     cart.group ||= "TEST"
 
     # To delete file when task is completed

data/lib/rivendell/import/base.rb

@@ -22,12 +22,15 @@ module Rivendell::Import
       workers << Worker.new(self).start unless options[:dry_run]
 
       Rivendell::Import.logger.info "Listen files in #{directory}"
-      Listen.to(directory) do |modified, added, removed|
+
+      callback = Proc.new do |modified, added, removed|
         added.each do |file|
           Rivendell::Import.logger.debug "Detected file '#{file}'"
           file(file, directory)
         end
       end
+
+      Listen.to(directory).change(&callback).start!
     end
 
     def process(*paths)

data/lib/rivendell/import/cart.rb

@@ -54,7 +54,7 @@ module Rivendell::Import
     end
 
     def find_by_title(string, options = {})
-      if remote_cart = carts_cache.find_by_title(string, options)
+      if remote_cart = cart_finder.find_by_title(string, options)
         self.number = remote_cart.number
       end
     end
@@ -66,10 +66,21 @@ module Rivendell::Import
       self.clear_cuts = true
     end
 
-    def carts_cache
-      @carts_cache ||= Rivendell::Import::CartsCache.new(xport)
+    @db_url = nil
+    cattr_accessor :db_url
+
+    def cart_finder
+      @cart_finder ||= 
+        begin
+          unless db_url 
+            Rivendell::Import::CartFinder::ByApi.new xport
+          else
+            Rivendell::DB.establish_connection(db_url)
+            Rivendell::Import::CartFinder::ByDb.new
+          end
+        end
     end
-
+    
   end
 end
   

data/lib/rivendell/import/cart_finder.rb

@@ -0,0 +1,62 @@
+module Rivendell::Import
+  class CartFinder
+
+    def find_all_by_title(string, options = {})
+      normalizer = options.delete(:normalizer) || Proc.new { |id| id }
+
+      string = normalizer.call(string)
+      carts(options).select do |cart|
+        normalizer.call(cart.title) == string
+      end
+    end
+    
+    def default_normalizer
+      Proc.new do |string|
+        ActiveSupport::Multibyte::Chars.new(string).normalize(:kd).gsub(/[^\x00-\x7F]/n,'').downcase.to_s.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
+      end
+    end
+
+    def find_by_title(string, options = {})
+      matching_carts = find_all_by_title(string, options)
+
+      if matching_carts.blank?
+        matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
+      end
+
+      matching_carts.first if matching_carts.one?
+    end
+
+    class ByApi < CartFinder
+
+      attr_accessor :xport
+
+      def initialize(xport)
+        @xport = xport
+      end
+
+      def cache
+        clear if purged_at < time_to_live.ago
+        @cache ||= {}
+      end
+
+      def clear
+        @cache = nil
+      end
+
+      @@time_to_live = 600
+      cattr_accessor :time_to_live
+      
+      attr_accessor :purged_at
+      
+      def purged_at
+        @purged_at ||= Time.now
+      end
+      
+      def carts(options = {})
+        cache[options.to_s] ||= xport.list_carts(options)
+      end
+
+    end
+
+  end
+end

data/lib/rivendell/import/cart_finder_by_db.rb

@@ -0,0 +1,18 @@
+# Find Carts using a direct database access.
+# 
+# Provided to workaround actual accent problems experienced with Rivendell API.
+module Rivendell::Import
+  class CartFinder::ByDb < CartFinder
+  
+    def carts(options = {})
+      query_options = { :fields => [ :number, :title ] }
+      
+      if options[:group] 
+        query_options[:group_name] = options[:group] 
+      end
+
+      Rivendell::DB::Cart.all(query_options)
+    end
+  
+  end
+end

data/lib/rivendell/import/cli.rb

@@ -46,7 +46,7 @@ module Rivendell::Import
 
     def parser
       @parser ||= Trollop::Parser.new do
-        opt :config, "Configuration file", :type => String
+        opt :config, "Configuration file", :type => String, :required => true
         opt :listen, "Wait for files in given directory"
         opt :dry_run, "Just create tasks without executing them"
         opt :debug, "Enable debug messages (in stderr)"
@@ -78,7 +78,7 @@ module Rivendell::Import
       parsed_parser.leftovers
     end
 
-    def setup_logger 
+    def setup_logger
       new_logger =
         if syslog?
           Syslog::Logger.new('rivendell-import').tap do |syslog|
@@ -115,7 +115,7 @@ module Rivendell::Import
         Daemons.daemonize :app_name => "rivendell-import", :dir => pid_directory, :dir_mode => :normal
       end
     end
-      
+
     def run
       setup_logger
 
@@ -131,7 +131,7 @@ module Rivendell::Import
       config_loader.listen_file
 
       if listen_mode?
-        start_webserver 
+        start_webserver
 
         listen_options = {}
         listen_options[:dry_run] = true if dry_run?

data/lib/rivendell/import/config_loader.rb

@@ -19,12 +19,12 @@ module Rivendell::Import
       callback = Proc.new do |modified, added, removed|
         if modified.include? absolute_path
           Rivendell::Import.logger.info "Configuration changed, reload it"
-          load 
+          load
         end
       end
 
       Rivendell::Import.logger.info "Listen to config file changes (#{file})"
-      Listen.to(directory).filter(/^#{basename}$/).change(&callback).start(false)
+      Listen.to(directory).filter(/^#{basename}$/).change(&callback).start
     end
 
     def listen_file_with_inotify

data/lib/rivendell/import/task.rb

@@ -99,5 +99,15 @@ module Rivendell::Import
       save!
     end
 
+    after_create :purge!
+
+    def purge!
+      self.class.purge!
+    end
+
+    def self.purge!
+      where("created_at < ?", 24.hours.ago).destroy_all
+    end
+
   end
 end

data/lib/rivendell/import/version.rb

@@ -1,5 +1,5 @@
 module Rivendell
   module Import
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/lib/rivendell/import.rb

@@ -40,7 +40,7 @@ require "rivendell/import/worker"
 require "rivendell/import/task"
 require "rivendell/import/tasks"
 require "rivendell/import/base"
-require "rivendell/import/carts_cache"
+require "rivendell/import/cart_finder"
 require "rivendell/import/cart"
 require "rivendell/import/context"
 require "rivendell/import/cut"
@@ -48,3 +48,7 @@ require "rivendell/import/file"
 require "rivendell/import/notification"
 require "rivendell/import/notifier/base"
 require "rivendell/import/notifier/mail"
+
+# TODO Rivendell::Import::CartFinder::ByDb should be optional
+require 'rivendell/db'
+require 'rivendell/import/cart_finder_by_db'

data/rivendell-import.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Rivendell::Import::VERSION
 
-  gem.add_runtime_dependency 'listen'
+  gem.add_runtime_dependency 'listen', '~> 1.2.1'
   gem.add_runtime_dependency 'httmultiparty'
   gem.add_runtime_dependency 'rivendell-api', '~> 0.0.4'
   gem.add_runtime_dependency 'trollop'
@@ -30,6 +30,8 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'daemons'
 
+  gem.add_runtime_dependency 'rivendell-db'
+
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "guard"

data/spec/rivendell/import/{carts_cache_spec.rb → cart_finder_spec.rb}

@@ -1,18 +1,13 @@
+# -*- coding: utf-8 -*-
 require 'spec_helper'
 
-describe Rivendell::Import::CartsCache do
-
-  let(:carts_cache) { Rivendell::Import::CartsCache.new(xport) }
-  subject { carts_cache }
+describe Rivendell::Import::CartFinder do
 
   let(:cart) { mock :title => "dummy" }
   let(:carts) { [ cart ] }
 
-  let(:xport) { mock }
-
-  before do
-    subject.stub :xport => xport
-  end
+  let(:cart_finder) { Rivendell::Import::CartFinder.new }
+  subject { cart_finder }
 
   describe "#find_all_by_title" do
 
@@ -41,7 +36,7 @@ describe Rivendell::Import::CartsCache do
 
   describe "#default_normalizer" do
 
-    subject { carts_cache.default_normalizer }
+    subject { cart_finder.default_normalizer }
 
     it "should downcase string" do
       subject.call("ABC").should == "abc"
@@ -54,9 +49,50 @@ describe Rivendell::Import::CartsCache do
     it "should remove double spaces" do
       subject.call("a  b   c").should == "a b c"
     end
+
+    it "should remove accents" do
+      subject.call("Ç'est évident").should == "c est evident"
+    end
+    
+  end
+
+  describe "#find_by_title" do
+
+    before do
+      subject.stub(:carts).and_return(carts)
+    end
     
+    it "should try an exact match" do
+      subject.find_by_title(cart.title).should == cart
+    end
+
+    it "should try a match with default normalizer" do
+      subject.find_by_title(cart.title.upcase).should == cart
+    end
+
+    it "should return nil when no cart matchs" do
+      subject.find_by_title("nothing").should be_nil
+    end
+
+    it "should return nil when several carts match" do
+      carts << cart
+      subject.find_by_title(cart.title).should be_nil
+    end
+
   end
 
+end
+
+describe Rivendell::Import::CartFinder::ByApi do
+
+  let(:cart_finder) { Rivendell::Import::CartFinder::ByApi.new(xport) }
+  subject { cart_finder }
+
+  let(:xport) { mock }
+
+  let(:cart) { mock :title => "dummy" }
+  let(:carts) { [ cart ] }
+
   describe "#carts" do
 
     before do
@@ -82,27 +118,29 @@ describe Rivendell::Import::CartsCache do
 
   end
 
-  describe "#find_by_title" do
+end
+
+describe Rivendell::Import::CartFinder::ByDb do
+
+  let(:cart_finder) { Rivendell::Import::CartFinder::ByDb.new }
+  subject { cart_finder }
+
+  let(:cart) { mock :title => "dummy" }
+  let(:carts) { [ cart ] }
+
+  describe "#carts" do
 
     before do
-      subject.stub(:carts).and_return(carts)
+      Rivendell::DB::Cart.stub :all => carts
     end
     
-    it "should try an exact match" do
-      subject.find_by_title(cart.title).should == cart
-    end
-
-    it "should try a match with default normalizer" do
-      subject.find_by_title(cart.title.upcase).should == cart
-    end
-
-    it "should return nil when no cart matchs" do
-      subject.find_by_title("nothing").should be_nil
+    it "should return Carts found in database" do
+      subject.carts.should == carts
     end
 
-    it "should return nil when several carts match" do
-      carts << cart
-      subject.find_by_title(cart.title).should be_nil
+    it "should search in specified group if present" do
+      Rivendell::DB::Cart.should_receive(:all).with(hash_including(:group_name => "dummy")).and_return(carts)
+      subject.carts(:group => "dummy")
     end
 
   end

data/spec/rivendell/import/task_spec.rb

@@ -280,4 +280,25 @@ describe Rivendell::Import::Task do
 
   end
 
+  describe ".purge!" do
+    
+    it "should remove tasks older than 24 hours" do
+      old_task = Rivendell::Import::Task.create! :file => file, :created_at => 25.hours.ago
+      Rivendell::Import::Task.purge!
+      Rivendell::Import::Task.exists?(old_task).should be_false
+    end
+
+    it "should keep recent tasks" do
+      task = Rivendell::Import::Task.create! :file => file
+      Rivendell::Import::Task.purge!
+      Rivendell::Import::Task.exists?(task).should be_true
+    end
+
+    it "should be invoked each time a new Task is created" do
+      Rivendell::Import::Task.should_receive(:purge!)
+      Rivendell::Import::Task.create! :file => file
+    end
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rivendell-import
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-01 00:00:00.000000000 Z
+date: 2013-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: listen
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: httmultiparty
   requirement: !ruby/object:Gem::Requirement
@@ -203,6 +203,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rivendell-db
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -364,7 +380,8 @@ files:
 - lib/rivendell/import/application.rb
 - lib/rivendell/import/base.rb
 - lib/rivendell/import/cart.rb
-- lib/rivendell/import/carts_cache.rb
+- lib/rivendell/import/cart_finder.rb
+- lib/rivendell/import/cart_finder_by_db.rb
 - lib/rivendell/import/cli.rb
 - lib/rivendell/import/config.rb
 - lib/rivendell/import/config_loader.rb
@@ -393,8 +410,8 @@ files:
 - rivendell-import.gemspec
 - spec/fixtures/mail-body.erb
 - spec/rivendell/import/base_spec.rb
+- spec/rivendell/import/cart_finder_spec.rb
 - spec/rivendell/import/cart_spec.rb
-- spec/rivendell/import/carts_cache_spec.rb
 - spec/rivendell/import/cli_spec.rb
 - spec/rivendell/import/config_spec.rb
 - spec/rivendell/import/context_spec.rb
@@ -429,7 +446,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1619764904591619983
+      hash: 3691791623266021807
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -438,7 +455,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1619764904591619983
+      hash: 3691791623266021807
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -453,8 +470,8 @@ test_files:
 - features/support/mock_xport.rb
 - spec/fixtures/mail-body.erb
 - spec/rivendell/import/base_spec.rb
+- spec/rivendell/import/cart_finder_spec.rb
 - spec/rivendell/import/cart_spec.rb
-- spec/rivendell/import/carts_cache_spec.rb
 - spec/rivendell/import/cli_spec.rb
 - spec/rivendell/import/config_spec.rb
 - spec/rivendell/import/context_spec.rb

data/lib/rivendell/import/carts_cache.rb

@@ -1,58 +0,0 @@
-module Rivendell::Import
-  class CartsCache
-
-    attr_accessor :xport
-
-    def initialize(xport)
-      @xport = xport
-    end
-
-    def find_all_by_title(string, options = {})
-      normalizer = options.delete(:normalizer) || Proc.new { |id| id }
-
-      string = normalizer.call(string)
-      carts(options).select do |cart|
-        normalizer.call(cart.title) == string
-      end
-    end
-    
-    def default_normalizer
-      Proc.new do |string|
-        string.downcase.gsub(/[^a-z0-9]/," ").gsub(/[ ]+/," ")
-      end
-    end
-
-    def find_by_title(string, options = {})
-      matching_carts = find_all_by_title(string, options)
-
-      if matching_carts.blank?
-        matching_carts = find_all_by_title(string, options.merge(:normalizer => default_normalizer))
-      end
-
-      matching_carts.first if matching_carts.one?
-    end
-
-    def cache
-      clear if purged_at < time_to_live.ago
-      @cache ||= {}
-    end
-
-    def clear
-      @cache = nil
-    end
-
-    @@time_to_live = 600
-    cattr_accessor :time_to_live
-
-    attr_accessor :purged_at
-
-    def purged_at
-      @purged_at ||= Time.now
-    end
-
-    def carts(options = {})
-      cache[options.to_s] ||= xport.list_carts(options)
-    end
-
-  end
-end