crabfarm 0.0.19 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90b7726fe962f1aa743078e2a50f6a477aab9e4d
-  data.tar.gz: ed95f4776e57812c16f10785cb587d2a02a7bd9a
+  metadata.gz: f93ecc74ca65159efaed6eeeb3ff85abbb13fe78
+  data.tar.gz: 1b1690612566628e7a9d3a8a8d8adef8e8c9291b
 SHA512:
-  metadata.gz: bf81855a24468d1ac95a0c8ce452385117c02e5fc3c115b7cb74ce51bd0cc66e8e441fece4db041f41f718ba808444115b2f032aa1e131558c05e8b816cd8912
-  data.tar.gz: 19f2b2f80c599b39a3fabb76065f1eacac2449e6d8f468ff595d9ee871ccdd7731d1902cf2f05668a7bd3c8f642a0b76a081987bb009dcf04e60767ea34812d3
+  metadata.gz: 130fee56339b38ccead67d986354622e6608c5cdc5f703ff6a5791d3b0995e88750163888cfdb09017522b67e369745b2a5e5949048905b8a81ac1b8098f82fb
+  data.tar.gz: 550b31026059c8a31795eab00f0928d22d773adeab91ab16b5ae9b2b83534677292a60bb1cfd6b85f3fe23d2a19214a7424a8ca30ebd418e9f9ab02c449645a5

data/lib/crabfarm/assertion/context.rb

@@ -0,0 +1,13 @@
+require "crabfarm/assertion/wrapper"
+
+module Crabfarm
+  module Assertion
+    module Context
+
+      def assert(_value)
+        Wrapper.new _value, self
+      end
+
+    end
+  end
+end

data/lib/crabfarm/assertion/parsers.rb

@@ -0,0 +1,105 @@
+module Crabfarm
+  module Assertion
+    module Parsers
+
+      def parse_integer _value, _options={}
+        parse_number :to_i, _value, _options
+      end
+
+      def parse_float _value, _options={}
+        parse_number :to_f, _value, _options
+      end
+
+      def parse_phrase _value, _options={}
+        fail_with_nil if _value.nil?
+
+        if _value.is_a? String
+          clean_string _value, _options
+        else
+          if _value.respond_to? :to_s
+            clean_string _value.to_s, _options
+          else
+            fail_with "Value cannot be transformed to string"
+          end
+        end
+      end
+
+      def parse_boolean _value, _options={}
+        truthy = Array(_options.fetch(:truthy, ['true']))
+        falsy = Array(_options.fetch(:falsy, ['false']))
+
+        mapping = { :empty => false, true => true, false => false }
+        truthy.each { |k| mapping[k] = true }
+        falsy.each { |k| mapping[k] = false }
+
+        parse_by_map _value, mapping
+      end
+
+      def parse_by_map _value, _map={}
+        fail_with_nil if _value.nil?
+
+        case _value
+        when String, Symbol
+          _value = _value.strip if _value.is_a? String
+          return parse_by_map :empty, _map if _value.empty?
+          fail_with "#'{_value.to_s}' is and invalid keyword" unless _map.key? _value
+          _map[_value]
+        else
+          fail_with "'#{_value}' cannot be mapped"
+        end
+      end
+
+      def parse_number _to_fun, _value, _options={}
+        fail_with_nil if _value.nil?
+
+        if _value.is_a? String
+          _value.strip!
+          if _value.empty?
+            return_default _options
+          else
+            extract_number_from_string(_value, _options).send _to_fun
+          end
+        elsif _value.respond_to? _to_fun
+          _value.send _to_fun
+        else
+          fail_with "'#{_value}' cannot be transformed to number"
+        end
+      end
+
+      def clean_string _string, _options
+        normalized = _string.strip.gsub(/\s+/, ' ')
+        if normalized.empty?
+          return_default _options
+        else normalized end
+      end
+
+      def fail_with_nil
+        fail_with "'nil' cant be parsed"
+      end
+
+      def return_default _options
+        fail_with "Value is an empty" unless _options.key? :default
+        _options[:default]
+      end
+
+      def extract_number_from_string _value, _options
+        decimal_mark = _options.fetch(:decimal_mark, '.') # TODO: make default decimal mark configurable
+        thousand_mark = _options.fetch(:thousand_mark, infer_thousand_separator(decimal_mark))
+        num_rgx = /(?:\.\d+|\d{1,3}(?:\,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)/
+        num_rgx = Regexp.new "(?:\\#{decimal_mark}\\d+|\\d{1,3}(?:\\#{thousand_mark}\\d{3})+(?:\\#{decimal_mark}\\d+)?|\\d+(?:\\#{decimal_mark}\\d+)?)"
+        matches = _value.scan num_rgx
+        fail_with "'#{_value}' has an ambiguous numeric format" if matches.count > 1
+        fail_with "'#{_value}' does not contain any number" if matches.count < 1
+        matches.first.gsub(thousand_mark,'').gsub(decimal_mark, '.')
+      end
+
+      def infer_thousand_separator _decimal_mark
+        case _decimal_mark
+        when '.' then ','
+        when ',' then '.'
+        else '' end
+      end
+
+    end
+  end
+end

data/lib/crabfarm/assertion/validations.rb

@@ -0,0 +1,28 @@
+module Crabfarm
+  module Assertion
+    module Validations
+
+      def validate_number _value, _options={}
+        fail_with "#{_value} out of range" if _options.key? :greater_than and _value <= _options[:greater_than]
+        fail_with "#{_value} out of range" if _options.key? :greater_or_equal_to and _value < _options[:greater_or_equal_to]
+        fail_with "#{_value} out of range" if _options.key? :less_than and _value >= _options[:less_than]
+        fail_with "#{_value} out of range" if _options.key? :less_or_equal_to and _value > _options[:less_or_equal_to]
+        fail_with "#{_value} out of range" if _options.key? :between and not _options[:between].include? _value
+      end
+
+      def validate_word _value, _options={}
+        fail_with "'#{_value}' is not a single word" if /\s/ === _value
+        validate_string _value, _options
+      end
+
+      def validate_string _value, _options={}
+        fail_with "#{_value} does not match expression" if _options.key? :matches and not _options[:matches] === _value
+      end
+
+      def validate_general _value, _options={}
+        fail_with "#{_value} is not recognized" if _options.key? :in and not _options[:in].include? _value
+      end
+
+    end
+  end
+end

data/lib/crabfarm/assertion/wrapper.rb

@@ -0,0 +1,101 @@
+require "crabfarm/assertion/parsers"
+require "crabfarm/assertion/validations"
+
+module Crabfarm
+  module Assertion
+    class Wrapper
+      include Parsers
+      include Validations
+
+      def initialize(_value, _context=nil)
+        @value = if _value.respond_to? :text
+          _value.text
+        else _value end
+
+        @context = _context
+      end
+
+      def is_integer(_options={})
+        perform_assertion(_options) {
+          @value = parse_integer @value, _options
+          validate_number @value, _options
+          validate_general @value, _options
+        }
+      end
+
+      alias :is_i :is_integer
+
+      def is_float(_options={})
+        perform_assertion(_options) {
+          @value = parse_float @value, _options
+          validate_number @value, _options
+          validate_general @value, _options
+        }
+      end
+
+      alias :is_f :is_float
+
+      def is_word(_options={})
+        perform_assertion(_options) {
+          @value = parse_phrase @value, _options
+          validate_word @value, _options
+          validate_general @value, _options
+        }
+      end
+
+      alias :is_w :is_word
+      def is_string(_options={})
+        perform_assertion(_options) {
+          @value = parse_phrase @value, _options
+          validate_string @value, _options
+          validate_general @value, _options
+        }
+      end
+
+      alias :is_s :is_string
+
+      def is_boolean(_options={})
+        perform_assertion(_options) {
+          @value = parse_boolean @value, _options
+        }
+      end
+
+      alias :is_b :is_boolean
+
+      def matches(_rgx, _options={})
+        perform_assertion(_options) {
+          match = _rgx.match @value
+          fail_with "#{@value} does not match #{_rgx.to_s}" if match.nil?
+          match
+        }
+      end
+
+      def method_missing(_method, *_args)
+        if @value.respond_to? :method
+          @value = @value.send(*_args)
+        else super end
+      end
+
+      def respond_to?(*args)
+        @value.respond_to?(*args)
+      end
+
+    private
+
+      def perform_assertion(_options)
+        begin
+          yield
+          @value
+        rescue AssertionError # => exc
+          # for now just raise, in the future event could be hooked here...
+          raise
+        end
+      end
+
+      def fail_with(_message)
+        raise AssertionError.new _message
+      end
+
+    end
+  end
+end

data/lib/crabfarm/base_parser.rb

@@ -1,5 +1,8 @@
+require "crabfarm/assertion/context"
+
 module Crabfarm
   class BaseParser < Delegator
+    include Assertion::Context
 
     attr_reader :params, :document
 

data/lib/crabfarm/base_state.rb

@@ -1,8 +1,10 @@
 require 'thwait'
 require 'crabfarm/forked_state'
+require "crabfarm/assertion/context"
 
 module Crabfarm
   class BaseState
+    include Assertion::Context
     extend Forwardable
 
     PARSE_METHOD_RX = /^parse_(.*)$/

data/lib/crabfarm/cli.rb

@@ -75,9 +75,12 @@ module Crabfarm
 
       c.desc "Generates a new crabfarm application"
       c.command :app do |app|
+        app.desc "Set the remote used by the crawler"
+        app.flag [:r, :remote]
+
         app.action do |global_options,options,args|
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.new.generate_app(args[0], Dir.pwd)
+          Crabfarm::Modes::Generator.generate_app(Dir.pwd, args[0], options[:remote])
         end
       end
 
@@ -85,7 +88,7 @@ module Crabfarm
       c.command :parser do |parser|
         parser.action do |global_options,options,args|
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.new.generate_parser(args[0])
+          Crabfarm::Modes::Generator.generate_parser(args[0])
         end
       end
 
@@ -93,7 +96,7 @@ module Crabfarm
       c.command :state do |parser|
         parser.action do |global_options,options,args|
           require "crabfarm/modes/generator"
-          Crabfarm::Modes::Generator.new.generate_state(args[0])
+          Crabfarm::Modes::Generator.generate_state(args[0])
         end
       end
     end
@@ -108,7 +111,7 @@ module Crabfarm
       end
     end
 
-    desc "Publish the crawler to a crabfarm cloud"
+    desc "Publish the crawler to the crabfarm.io cloud"
     command :publish do |c|
       c.desc "Just list the files that are beign packaged"
       c.switch :dry, :default_value => false
@@ -119,6 +122,8 @@ module Crabfarm
       c.action do |global_options,options,args|
         next puts "This command can only be run inside a crabfarm application" unless defined? CF_PATH
 
+        options[:remote] = args[0]
+
         require "crabfarm/modes/publisher"
         Crabfarm::Modes::Publisher.publish CF_PATH, options
       end

data/lib/crabfarm/errors.rb

@@ -4,6 +4,8 @@ module Crabfarm
 
   class ConfigurationError < Error; end
 
+  class AssertionError < Error; end
+
   class EntityNotFoundError < Error
     attr_accessor :role, :name
 

data/lib/crabfarm/modes/generator.rb

@@ -6,12 +6,13 @@ require 'ostruct'
 
 module Crabfarm
   module Modes
-    class Generator
+    module Generator
 
-      def generate_app(_name, _target)
+      def generate_app(_target, _name, _default_remote=nil)
         with_external_path _target do
           binding = {
             name: _name,
+            remote: _default_remote,
             version: Crabfarm::VERSION
           }
 
@@ -83,7 +84,7 @@ module Crabfarm
         self
       end
 
-  private
+    private
 
       def generate_dir(_path, _silent)
         path = File.join(*_path)
@@ -123,6 +124,8 @@ module Crabfarm
       def render_op(_op, _message, _color)
         puts _op.rjust(10).color(_color) + '  ' + _message
       end
+
+      extend self
     end
   end
 end

data/lib/crabfarm/modes/publisher.rb

@@ -1,8 +1,10 @@
 require 'yaml'
+require 'json'
 require 'git'
 require 'zlib'
+require 'inquirer'
 require 'rubygems/package'
-require 'net/http/post/multipart'
+require 'base64'
 require 'rainbow'
 require 'rainbow/ext/string'
 require 'digest/sha1'
@@ -20,7 +22,7 @@ module Crabfarm
         @options = _options
 
         load_config
-        return unless dry_run or authenticated?
+        return unless dry_run or check_credentials
         detect_git_repo
 
         if inside_git_repo?
@@ -37,7 +39,7 @@ module Crabfarm
         compress_package
         generate_signature
 
-        send_package unless dry_run
+        send_package if not dry_run and ensure_valid_remote
       end
 
     private
@@ -59,27 +61,76 @@ module Crabfarm
       end
 
       def load_config
-        config = YAML.load_file config_path
+        @local_config = YAML.load_file config_path
 
-        if File.exists? home_config_path
-          home_config = YAML.load_file home_config_path
-          config = home_config.merge config
-        end
+        @home_config = if File.exists? home_config_path
+          YAML.load_file home_config_path
+        else {} end
+
+        config = @home_config.merge @local_config
 
         @token = config['token']
-        @name = config['name']
+        @url = @options[:remote] || config['remote']
         @host = config['host'] || DEFAULT_HOST
         @include = config['files']
       end
 
-      def authenticated?
-        # TODO: if no token, ask for credentials and fetch token
-        if @token.nil? or @token.empty?
-          puts "No crabfarm API token has been provided".color(:red)
-          return false
+      def ensure_valid_remote
+        if @url.nil?
+          @url = Ask.input 'Enter default remote for crawler'
+          return false unless validate_remote @url
+          @local_config['remote'] = @url
+          save_local_config
+          return true
+        else
+          validate_remote @url
         end
+      end
+
+      def validate_remote(_url)
+        return true if /^\w+\/\w+$/i === _url
+        puts "Invalid remote syntax: #{_url}".color :red
+        return false
+      end
+
+      def check_credentials
+        if @token.nil?
+          puts 'No credential data found, please identify yourself'
+          email = Ask.input 'Enter your crabfarm.io email'
+          password = Ask.input 'Enter your crabfarm.io password'
+
+          resp = send_request Net::HTTP::Post, 'api/tokens', {
+            'email' => email,
+            'password' => password
+          }
+
+          case resp
+          when Net::HTTPCreated
+            @token = JSON.parse(resp.body)['token']
+            @home_config['token'] = @token
+            save_home_config
+          when Net::HTTPUnauthorized
+            puts "The provided credentials are invalid!".color(:red)
+          else
+            puts "Unknown error when asking for token!".color(:red)
+          end
+        end
+
+        not @token.nil?
+      end
+
+      def save_local_config
+        save_config config_path, @local_config
+      end
 
-        true
+      def save_home_config
+        save_config home_config_path, @home_config
+      end
+
+      def save_config(_path, _config)
+        data = YAML.dump _config
+        data = data.split("\n", 2).last # remove first line to make it more readable
+        File.open(_path, 'w') { |f| f.write data }
       end
 
       def is_tree_dirty?
@@ -167,21 +218,35 @@ module Crabfarm
       end
 
       def send_package
-        url = URI.join(@host, 'api/crawlers/', @name)
-
-        req = Net::HTTP::Put::Multipart.new(url.path, {
-          "repo" => UploadIO.new(StringIO.new(@cpackage.string), "application/x-gzip", "tree.tar.gz"),
+        resp = send_request(Net::HTTP::Put, "api/bots/#{@url}", {
+          "repo" => Base64.encode64(@cpackage.string),
           "sha" => @signature,
           "ref" => @ref
-        }, {
-          'X-Api-Token' => @token
         })
 
-        res = Net::HTTP.start(url.host, url.port) do |http|
-          http.request(req)
+        case resp
+        when Net::HTTPSuccess
+          sha = JSON.parse(resp.body)['sha']
+          puts "#{@url} updated!"
+        when Net::HTTPUnauthorized
+          puts "You are not authorized to update crawler: #{@url}".color(:red)
+        when Net::HTTPNotFound
+          puts "Crawler not found: #{@url}".color(:red)
+        else
+          puts "Unknown error when updating crawler information!".color(:red)
         end
+      end
+
+      def send_request(_class, _path, _data=nil)
+        uri = URI.join(@host, _path)
 
-        puts res.body
+        req = req = _class.new uri.path
+        req.set_form_data _data
+        req['X-User-Token'] = @token unless @token.nil?
+
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          http.request(req)
+        end
       end
 
     end

data/lib/crabfarm/templates/dot_crabfarm.erb

@@ -1,9 +1,9 @@
-host: 'http://www.crabfarm.io'
-name: '<%= name %>'
+host: http://api.crabfarm.io
+<% unless remote.nil? %>remote: <%= remote %><% end %>
 files:
 - Crabfile
 - Gemfile
 - Gemfile.lock
 - boot.rb
 - app/**/*.*
-- bin/**/*.*
+- bin/**/*.*

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.19"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.19
+  version: 0.1.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-16 00:00:00.000000000 Z
+date: 2015-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
@@ -72,6 +72,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: inquirer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
@@ -128,20 +142,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: multipart-post
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -338,6 +338,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.7
 description: 
 email:
 - ignacio@platan.us
@@ -354,6 +368,10 @@ files:
 - lib/crabfarm/adapters/output/ostruct.rb
 - lib/crabfarm/adapters/parser/nokogiri.rb
 - lib/crabfarm/adapters/parser/pdf_reader.rb
+- lib/crabfarm/assertion/context.rb
+- lib/crabfarm/assertion/parsers.rb
+- lib/crabfarm/assertion/validations.rb
+- lib/crabfarm/assertion/wrapper.rb
 - lib/crabfarm/base_parser.rb
 - lib/crabfarm/base_state.rb
 - lib/crabfarm/cli.rb
@@ -369,9 +387,7 @@ files:
 - lib/crabfarm/dsl/surfer.rb
 - lib/crabfarm/engines/safe_state_loop.rb
 - lib/crabfarm/errors.rb
-- lib/crabfarm/event_store.rb
 - lib/crabfarm/forked_state.rb
-- lib/crabfarm/helpers/value_helper.rb
 - lib/crabfarm/http_client.rb
 - lib/crabfarm/mocks/noop_driver.rb
 - lib/crabfarm/modes/console.rb

data/lib/crabfarm/event_store.rb

@@ -1,20 +0,0 @@
-module Crabfarm
-  class EventStore
-
-    def initialize
-      @events = []
-      @mutex = Mutex.new
-    end
-
-    def event(_category, _message)
-      @mutex.synchronize do
-        @events << {
-          created_at: Time.current,
-          category: _category,
-          msg: _message
-        }
-      end
-    end
-
-  end
-end

data/lib/crabfarm/helpers/value_helper.rb

@@ -1,131 +0,0 @@
-module Crabfarm
-  module AssertionHelper
-
-
-    class Assert
-
-    assert_int('20', min: -10, max: 4, in: [1,2,3,4] )
-    assert_float()
-    assert_boolean()
-    assert val, is: :integer|float, between: xxx, default: nil
-    assert val, is: :boolean
-
-    test = assert(val).is_i between: 1..2, empty: 20, in: [1, 2, 3], default: nil # default when empty
-    assert(val).is_b empty: 20 # trims, downcase
-    assert(val).trimmed.downcased.is_s in: ['test', 'peo', 'hola'], matches: /asadad/, empty: :nil|false|true
-    assert(products.count).is_i greater_than: 20
-
-
-
-
-     matches
-    assert(val).is_i between: 1..2, default: 20
-    assert(val).is_f
-    assert(val).is_s default: 'hello', in: ['servicentro', nil]
-    assert(val).is_boolean default: true, truthy: 'servicentro', falsy: ['falso', 'caca']
-    assert(val).downcased.is_one_of 'hola'
-    assert(val).is_not_nil
-    assert(val).is_of_size
-    assert(val).is_date 'YYYY-MM-dd'
-
-    class AssertionWrapper
-
-      def stripped
-        @value = @value.strip if @value.is_a? String
-        self
-      end
-
-      def trimmed
-        stripped
-        @value = @value.gsub(/\s+/, ' ') if @value.is_a? String
-        self
-      end
-
-      def upcased
-        @value = @value.upcase if @value.is_a? String
-        self
-      end
-
-      def downcased
-        @value = @value.downcase if @value.is_a? String
-        self
-      end
-
-      def is_integer(_options={})
-        perform_assertion {
-          @value = parse_integer @value, _options
-          validate_numeric @value, _options
-          validate_general @value, _options
-        }
-      end
-
-      alias :is_integer :is_i
-
-      def is_float(_options={})
-        perform_assertion {
-          @value = parse_float @value, _options
-          validate_numeric @value, _options
-          validate_general @value, _options
-        }
-      end
-
-      alias :is_float :is_f
-
-      def is_string(_options={})
-        perform_assertion {
-          @value = parse_string @value, _options
-          validate_general @value, _options
-        }
-      end
-
-      alias :is_string :is_s
-
-      def is_boolean(_options={})
-        perform_assertion {
-          @value = parse_boolean @value
-        }
-      end
-
-      alias :is_string :is_b
-
-      def matches(_rgx, _)
-        perform_assertion {
-
-        }
-      end
-
-    private
-
-      def perform_assertion
-      end
-
-      def parse_integer _val, _options
-      end
-
-      def parse_float _val, _options
-      end
-
-      def parse_string _val, _options
-      end
-
-      def parse_boolean _val, _options
-      end
-
-      def is_numeric(_options)
-      end
-
-      def is_
-
-      def handle_error(_assertion)
-      end
-
-    end
-
-
-    assert_date()
-    assert_not_nil()
-    assert_count( between: 0..2, nil: true, halt: true)
-    assert_regex( )
-
-  end
-end