gogdb 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82cfcbddbebad876b49fd5dd07b98524daefdce5
+  data.tar.gz: 1e1c39defc8346d307931e31dd453bff05011171
+SHA512:
+  metadata.gz: aad7dc8e62fe1867dfeda1d8505c0dc5156e88defa212916e2ff664bc2dc2461af0ad00a01793e6c5d0a67af04bbc98b0ac907a78ce3407a7d9cd2b71d13f57a
+  data.tar.gz: c3d4d03217dd48daee67bcd48eb52f435c8adcaccfe2c44a3b9910006be50300f24c0b7c83661af8d22a57a48bb064fadd6595a4299ac6d6a0a949664ba6934d

data/LICENSE

@@ -0,0 +1,17 @@
+Copyright (c) 2014 Rolandas Barysas
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.

data/README.md

@@ -0,0 +1,12 @@
+gogdb [![Build Status](https://travis-ci.org/rbrs/gogdb.svg?branch=develop)](https://travis-ci.org/rbrs/gogdb)
+=============
+
+gogdb is a web crawler for ![gog.com](http://gog.com).
+
+The purpose of this application is to gather data from gog.com and constantly update database with changes. It is being developed as a backend part for ![gogdb.net](https://github.com/rbrs/gogdb.net).
+
+**This gem is currently in development and is not ready for general use.**
+
+### License
+
+This code is free software; you can redistribute it and/or modify it under the terms of the zlib License. A copy of this license can be found in the included LICENSE file.

data/bin/gogdb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'gogdb'
+Gogdb::Cli.start(ARGV)

data/lib/gogdb.rb

@@ -0,0 +1,16 @@
+require 'thor'
+require 'colorize'
+require 'net/http'
+require 'net/ping'
+require 'json'
+require 'gogdb/version'
+require 'gogdb/engine'
+require 'gogdb/utils'
+require 'gogdb/logger'
+require 'gogdb/cli'
+
+module Gogdb
+  GOG_URL = "http://www.gog.com"
+  GOG_GAMES_URL = "http://www.gog.com/games"
+  GOG_MOVIES_URL = "http://www.gog.com/movies"
+end

data/lib/gogdb/cli.rb

@@ -0,0 +1,27 @@
+module Gogdb
+  
+  class Cli < Thor
+    package_name "Gogdb"
+    map "-L" => :list
+
+    desc "version", "Shows current version"
+    def version
+      puts Gogdb::VERSION
+    end
+
+    desc "sync [options]", "Syncs all data between databases"
+    def sync
+    end
+
+    desc "fetch [options]", "Fetches items from gog.com and updates database"
+    method_option :limit, :type => :numeric, :desc => "How many items to fetch [0 - no limit]", :default => 0
+    method_option :type, :type => :string, :enum => ["all", "games", "movies"], :desc => "Type items to fetch", :default => "all"
+    method_option :debug, :type => :boolean, :desc => "Show debug messages", :default => false
+    method_option :silent, :type => :boolean, :desc => "Hide all output", :default => false
+    def fetch
+      @e = Gogdb::Engine.new(options)
+      @e.fetch(options)
+    end
+
+  end
+end

data/lib/gogdb/engine.rb

@@ -0,0 +1,88 @@
+module Gogdb
+  class Engine
+
+    def initialize(options={})
+      @logger = Logger.new(options)
+      @utils = Utils.new(@logger)
+    end
+
+    # Downloads data from source and updates database with changes
+    #
+    # @param [Hash] options Additional options such as :limit and :type
+    def fetch(options)
+      url = GOG_GAMES_URL if options[:type] == "games"
+      url = GOG_MOVIES_URL if options[:type] == "movies"
+
+      pages_number = get_pages_number(url)
+      @logger.debug "Pages received: #{pages_number}"
+      nil if pages_number < 1 # Do nothing if there are no pages
+
+      # Loop through pages
+      count = 0;
+      catch :limitReached do
+        for i in 0..pages_number do
+          # Let's get products number and URL's for current page
+          items_data = get_data(url)['products']
+          items_number = items_data.length
+          
+          # Loop through products and get data of every one
+          for i in 0..items_number do
+            item_url = items_data[i]['url']
+            item_data = get_data("#{GOG_URL}#{item_url}")['gameProductData']
+
+            @logger.debug "Item received: #{item_data['title']}"
+
+            count += 1
+            # Break loop if limit is reached
+            throw :limitReached if count >= options[:limit] && options[:limit] != 0
+          end
+        end
+      end
+    end
+
+    # Gets pages number.
+    #
+    # @param [string] url Page URL
+    # @return [integer] number of pages
+    def get_pages_number(url)
+      begin
+        get_data(url)["totalPages"]
+      rescue => e
+        @logger.error "Cannot retrieve pages number."
+        @logger.debug e.message
+        0
+      end
+    end
+
+    # Gets data from source and parses global javascript object.
+    #
+    # @param [string] url Page URL
+    # @return [hash] 
+    def get_data(url)
+      begin
+        page = Net::HTTP.get(URI(url))
+        JSON.parse(page[/(?<=var gogData = )(.*)(?=;)/,1])
+      rescue => e
+        @logger.warning "Cannot retrieve or parse data from gog.com"
+        @logger.error e.message
+
+        # In case crawler cannot access gog.com, let's check if gog.com is
+        # actually online.
+        ph = Net::Ping::HTTP.new(url)
+        unless ph.ping?
+          @logger.warning "Cannot establish connection to gog.com. Retrying..."
+          
+          # Retry connection until gog.com comes online. After that - retry
+          # getting data.
+          @utils.retryConnection(url)
+          retry
+        else
+          @logger.warning "Connection to gog.com established successfully. Retrying..."
+          sleep(15)
+          retry
+        end
+      end
+    end
+
+  end
+end

data/lib/gogdb/logger.rb

@@ -0,0 +1,28 @@
+module Gogdb
+  class Logger
+    def initialize(options={})
+      @debug = options[:debug] || false
+      @silent = options[:silent] || false
+    end
+
+    def log(message)
+      puts message unless @silent
+    end
+
+    def error(message)
+      puts "[#{time}] #{message}".colorize(:light_red) unless @silent
+    end
+
+    def warning(message)
+      puts "[#{time}] #{message}".colorize(:light_yellow) unless @silent
+    end
+
+    def debug(message)
+      puts "[#{time}] #{message}".colorize(:light_blue) if @debug
+    end
+
+    def time
+      Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    end
+  end
+end

data/lib/gogdb/utils.rb

@@ -0,0 +1,30 @@
+module Gogdb
+  class Utils
+
+    def initialize(logger=Logger.new({}))
+      @logger = logger 
+    end
+
+    # Retries connection to GOG.com incrementally (every 10n seconds, up to 120)
+    #
+    # @params [String]
+    # @return [Boolean]
+    def retryConnection(url)
+      @count = 1
+      ph = Net::Ping::HTTP.new(url)
+      
+      while true do
+        if ph.ping?
+          @logger.warning "Connection to gog.com established successfully. Retrying previous task..."
+          true
+        else
+          @logger.error "Cannot establish connection. Retrying in #{@count * 10} seconds..."
+        end
+      
+        sleep(10 * @count)
+        @count += 1 if @count < 12
+      end
+    end
+
+  end
+end

data/lib/gogdb/version.rb

@@ -0,0 +1,3 @@
+module Gogdb
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: gogdb
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Rolandas Barysas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.20'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: The backend part for gogdb.net. Constantly looks for changes on gog.com.
+email:
+- r.b@riseup.net
+executables:
+- gogdb
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/gogdb
+- lib/gogdb.rb
+- lib/gogdb/cli.rb
+- lib/gogdb/engine.rb
+- lib/gogdb/logger.rb
+- lib/gogdb/utils.rb
+- lib/gogdb/version.rb
+homepage: http://www.gogdb.net
+licenses:
+- zlib
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Web crawler for gog.com
+test_files: []