skynet-core 0.0.3 → 0.0.4

data/Rakefile

@@ -1,16 +1,33 @@
 require 'rubygems'
 require 'rake'
-require 'echoe'
 
-Echoe.new('skynet-core', '0.0.3') do |s|
-  s.description     = "Datamining Bot"
-  s.url             = "http://github.com/a2c/skynet-core"
-  s.author          = "Henry Hamon"
-  s.email           = "henry.hamon@a2c.com.br"
-  s.ignore_pattern  = ["tmp/*", "script/*"]
-  s.runtime_dependencies = ["json", "nokogiri",
-    "mechanize", "activerecord", "simpleconsole"]
-  s.development_dependencies = ["rspec"]
-end
+# task :default => :gem
+# 
+# task :gem do
+  require 'echoe'
+  Echoe.new('skynet-core', '0.0.4') do |s|
+    s.description     = "Datamining Bot"
+    s.url             = "http://github.com/a2c/skynet-core"
+    s.author          = "Henry Hamon"
+    s.email           = "henry.hamon@a2c.com.br"
+    s.ignore_pattern  = ["tmp/*", "script/*"]
+    s.runtime_dependencies = ["json", "nokogiri", "mechanize", "activerecord"]
+    s.development_dependencies = ["rspec"]
+  end
 
-Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+  Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+# end
+# 
+# desc "Migrate the database through scripts in db/migrate. Target specific version with VERSION=x"
+# task :migrate => :environment do
+  # ActiveRecord::Migrator.migrate('lib/skynet-core/db/migrate', ENV["VERSION"] ? ENV["VERSION"].to_i : nil )
+# end
+# 
+# task :environment do
+  # require 'active_record'
+  # require 'yaml'
+# 
+  # database_config = YAML::load(File.open('config/database.yml'))
+  # ActiveRecord::Base.establish_connection database_config[ENV["ENV"] ? ENV["ENV"] : "development"]["skynet"]
+# end
+# 

data/config/database.yml.example

@@ -1,10 +1,6 @@
 development:
   skynet:
-    adapter: mysql
-    database: skynet_staging
-    username: root
-    password: toor
-    encoding: utf8
+    path: '/home/henry/workspace/db/skynet.db'
 
   ibrands:
     adapter: mysql
@@ -16,14 +12,7 @@ development:
 
 production:
   skynet:
-    adapter: mysql
-    encoding: utf8
-    database: skynet
-    username: skynet
-    password: "MXQuE4KW9NhjEr7t"
-    host: "vps3.clubedeautores.com.br"
-    port: 3300
-   # socket: /var/run/mysqld/mysqld.sock
+    path: '/home/henry/workspace/db/skynet.db'
 
   ibrands:
     adapter: mysql

data/lib/skynet-core/cyberdyne.rb

@@ -1,4 +1,5 @@
 require "#{SKYNET_ROOT}/lib/skynet-core/db/ibrands"
+require "#{SKYNET_ROOT}/lib/skynet-core/db/hippocampus"
 
 module Skynet
   extend self

data/lib/skynet-core/db/hippocampus.rb

@@ -1,12 +1,70 @@
 require File.expand_path("#{File.dirname(__FILE__)}/../../skynet-core")
-require 'activerecord'
+require 'xapian-fu'
+include XapianFu
 
-Skynet.load_config
-unless Skynet.database_config.nil?
-  ActiveRecord::Base.establish_connection( Skynet.database_config[Skynet.app_config['enviroment']]['skynet'] )
+module Skynet
+  extend self
+
+  class Queue
+    attr_accessor :url, :source, :post, :author, :description, :brand_id, :tracked_network_id,
+      :network_type, :slot_1, :slot_2, :slot_3, :integrated, :published_at,
+      :created_at
+
+    def initialize(attributes = nil)
+      self.attributes = attributes unless attributes.nil?
+    end
+
+    def save
+      output = false
+      if uniqueness?
+        Skynet.queue << to_hash
+        Skynet.queue.flush
+        output = true
+      end if required?
+      output
+    end
+
+    def self.create(attributes = nil)
+      if attributes.is_a?(Array)
+        attributes.collect { |attr| create(attr) }
+      else
+        object = new(attributes)
+        object.save
+        object
+      end
+    end
+
+    protected
+
+    def attributes=(value)
+      value.each_pair do |key, value|
+        begin
+          instance_variable_set("@#{key}", value)
+        rescue
+          puts 'attribute not found'
+        end
+      end
+    end
+
+    def to_hash
+      {
+        :url => self.url, :source => self.source , :post => self.post.nil? ? '' : self.post,
+        :author => self.author.nil? ? '' : self.author, :description => self.description.nil? ? '' : self.description,
+        :brand_id => self.brand_id ,
+        :tracked_network_id => self.tracked_network_id , :network_type => self.network_type ,
+        :slot_1 => self.slot_1.nil? ? 0 : self.slot_1, :slot_2 => self.slot_2.nil? ? 0 : self.slot_2,
+        :slot_3 => self.slot_3.nil? ? 0 : self.slot_3,
+        :integrated => 0, :published_at => self.published_at, :created_at => DateTime.now
+      }
+    end
+
+    def uniqueness?
+      Skynet.queue.search("url:#{self.url.gsub('+','>')} network_type:#{self.network_type}").size < 1
+    end
+
+    def required?
+      !self.tracked_network_id.nil? || !self.url.nil? || !self.network_type.nil?
+    end
 
-  class Memory < ActiveRecord::Base
-    validates_presence_of :url
-    validates_uniqueness_of :url, :scope => :tracked_network_id
   end
 end

data/lib/skynet-core/db/migrate/001_create_actives_and_logs.rb

@@ -1,9 +1,5 @@
-require 'rubygems'
-require 'activerecord'
-require 'lib/skynet/hippocampus'
-
-#ActiveRecord::Schema.define do
 class CreateMemoriesAndLogs < ActiveRecord::Migration
+  def self.up
 
     create_table :memories do |t|
       t.column :brand_id, :integer
@@ -32,5 +28,13 @@ class CreateMemoriesAndLogs < ActiveRecord::Migration
       table.column :created_at, :datetime
     end
 
-end
 
+  end
+
+  def self.down
+    remove_index :memories, [:url, :tracked_network_id]
+    drop_table :memories
+    drop_table :logs
+
+  end
+end

data/lib/skynet-core/harvest.rb

@@ -1,5 +1,7 @@
 require "#{SKYNET_ROOT}/lib/skynet-core"
 require "#{SKYNET_ROOT}/lib/skynet-core/db/ibrands"
+require "#{SKYNET_ROOT}/lib/skynet-core/db/hippocampus"
+
 require 'open-uri'
 
 module Skynet

data/lib/skynet-core.rb

@@ -1,6 +1,9 @@
 require "rubygems"
 require "yaml"
 
+require 'xapian-fu'
+include XapianFu
+
 SKYNET_ROOT = File.expand_path("#{File.dirname(__FILE__)}/..") unless defined?(SKYNET_ROOT)
 
 Dir[File.dirname(__FILE__) + '/skynet-core/*.rb'].each {|file| require file }
@@ -10,7 +13,7 @@ SKYNET_ENV = ENV["SKYNET_ENV"] ||= ENV["RACK_ENV"] ||= "development" unless defi
 #
 module Skynet
 
-  attr_reader :_app_config, :_database_config
+  attr_reader :_app_config, :_database_config, :_queue
 
   class ApplicationLoadError < RuntimeError #:nodoc:
   end
@@ -56,6 +59,10 @@ module Skynet
       @_database_config ||= set_database_config
     end
 
+    def queue
+      @_queue ||= load_queue_database
+    end
+
     def require_dependencies(*paths)
       # Extract all files to load
       files = paths.map { |path| Dir[path] }.flatten
@@ -107,5 +114,32 @@ module Skynet
       YAML::load(File.open(root("config/database.yml"))) if File.exist?(root("config/database.yml"))
     end
 
+    def load_queue_database
+
+      XapianDb.new(:dir => self.database_config[self.app_config['enviroment']]['skynet']['path'], :create => true,
+        :fields => {
+          :url => { :store => true },
+          :source => { :store => true },
+          :post => { :store => true },
+          :author => { :store => true },
+          :description => { :store => true },
+          :brand_id => { :type => Integer, :store => true },
+          :tracked_network_id => { :type => Integer, :store => true },
+          :network_type => { :type => Integer, :store => true },
+          :slot_1 => { :type => Integer, :store => true },
+          :slot_2 => { :type => Integer, :store => true },
+          :slot_3 => { :type => Integer, :store => true },
+          :integrated => { :type => Integer, :store => true },
+          :published_at => { :type => DateTime, :store => true },
+          :created_at => { :type => DateTime, :store => true }
+      }, :store => [
+          :url, :source, :post, :author, :description, :brand_id, :tracked_network_id,
+          :network_type, :slot_1, :slot_2, :slot_3, :integrated, :published_at,
+          :created_at
+      ]
+      ) unless @_database_config.nil? & self.database_config[self.app_config['enviroment']]['skynet'].nil?
+
+    end
+
   end # self
 end

data/skynet-core.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{skynet-core}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Henry Hamon"]
-  s.date = %q{2010-06-09}
+  s.date = %q{2010-06-11}
   s.description = %q{Datamining Bot}
   s.email = %q{henry.hamon@a2c.com.br}
   s.extra_rdoc_files = ["README.mkdn", "lib/skynet-core.rb", "lib/skynet-core/cyberdyne.rb", "lib/skynet-core/db/hippocampus.rb", "lib/skynet-core/db/ibrands.rb", "lib/skynet-core/db/migrate/001_create_actives_and_logs.rb", "lib/skynet-core/db/migrate/002_pimp_log.rb", "lib/skynet-core/db/migration.rb", "lib/skynet-core/extend_string.rb", "lib/skynet-core/harvest.rb"]
@@ -27,14 +27,12 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_runtime_dependency(%q<mechanize>, [">= 0"])
       s.add_runtime_dependency(%q<activerecord>, [">= 0"])
-      s.add_runtime_dependency(%q<simpleconsole>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
     else
       s.add_dependency(%q<json>, [">= 0"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<mechanize>, [">= 0"])
       s.add_dependency(%q<activerecord>, [">= 0"])
-      s.add_dependency(%q<simpleconsole>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
     end
   else
@@ -42,7 +40,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<mechanize>, [">= 0"])
     s.add_dependency(%q<activerecord>, [">= 0"])
-    s.add_dependency(%q<simpleconsole>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: skynet-core
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: false
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Henry Hamon
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-09 00:00:00 -03:00
+date: 2010-06-11 00:00:00 -03:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -74,24 +74,10 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: simpleconsole
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -101,7 +87,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id006
+  version_requirements: *id005
 description: Datamining Bot
 email: henry.hamon@a2c.com.br
 executables: []