stackoverflow 0.1.0 → 0.1.1

data/bin/so

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'stackoverflow'
+
+StackOverflow::Command.new.run
+

data/lib/stackoverflow.rb

@@ -1,4 +1,22 @@
 #!/usr/bin/env ruby
+#
+# Copyright (C) 2012 Xavier Antoviaque <xavier@antoviaque.org>
+#
+# This software's license gives you freedom; you can copy, convey,
+# propagate, redistribute and/or modify this program under the terms of
+# the GNU Affero General Public License (AGPL) as published by the Free
+# Software Foundation (FSF), either version 3 of the License, or (at your
+# option) any later version of the AGPL published by the FSF.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program in a file in the toplevel directory called
+# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
+#
 
 require 'rubygems'
 require 'open-uri'
@@ -39,16 +57,31 @@ module StackOverflow
 
     class DB
         def initialize
-            @db = SQLite3::Database.new("../data/stackoverflow.db")
+            @db     = SQLite3::Database.new(File.join(Dir.home, ".stackoverflow/stackoverflow.db"))
+            @db_idx = SQLite3::Database.new(File.join(Dir.home, ".stackoverflow/stackoverflow_idx.db"))
         end
 
         def search(search_string)
-            sql = "SELECT id, score, body, title FROM posts WHERE post_type_id=1 AND "
+            # Search on titles in the small index DB, to get the ids faster
+            sql = "SELECT id FROM questions WHERE "
             sub_sql = []
             for search_term in search_string.split(' ')
-                sub_sql << "title LIKE '%#{search_term}%"
+                sub_sql << "title LIKE '%#{search_term}%'"
             end
             sql += sub_sql.join(' AND ')
+
+            questions_ids = []
+            @db_idx.execute(sql) do |row|
+                questions_ids << row[0]
+            end
+
+            # Then retreive details from the main (large) DB
+            sql = "SELECT id, score, body, title FROM posts WHERE "
+            sub_sql = []
+            for question_id in questions_ids
+                sub_sql << "id=#{question_id}"
+            end
+            sql += sub_sql.join(' OR ')
             sql += ' ORDER BY score DESC'
 
             questions = []
@@ -63,8 +96,23 @@ module StackOverflow
         end
 
         def get_answers(question_id)
+            # Search on parent ids in the small index DB, to get the ids faster
+            sql = "SELECT id FROM answers WHERE parent_id=#{question_id}"
+            answers_ids = []
+            @db_idx.execute(sql) do |row|
+                answers_ids << row[0]
+            end
+
+            # Then retreive details from the main (large) DB
+            sql = "SELECT id, score, body FROM posts WHERE "
+            sub_sql = []
+            for answer_id in answers_ids
+                sub_sql << "id=#{answer_id}"
+            end
+            sql += sub_sql.join(' OR ')
+            sql += ' ORDER BY score DESC'
+
             answers = []
-            sql = "SELECT id, score, body FROM posts WHERE post_type_id=2 AND parent_id=#{question_id} ORDER BY score DESC"
             @db.execute(sql) do |row|
                 answers << { :id => row[0],
                              :score => row[1],
@@ -73,6 +121,104 @@ module StackOverflow
         end
     end
 
+    class DBUpdater
+        def initialize
+            @dir_path = File.join(Dir.home, ".stackoverflow")
+            @db_version_path = File.join(@dir_path, "db_version")
+
+            @remote_hostname = "foobbs.org"
+            @remote_path = "/tmp/so/"
+        end
+
+        def check_local_dir
+            Dir.mkdir(@dir_path) if not directory_exists?(@dir_path)
+        end
+
+        def get_db_version
+            db_version = 0
+            if file_exists?(@db_version_path)
+                File.open(@db_version_path, 'r') { |f| db_version = f.read().to_i }
+            end
+            db_version
+        end
+
+        def set_db_version(version_nb)
+            File.open(@db_version_path, 'w+') { |f| f.write(version_nb.to_s) }
+        end
+
+        def get_remote_db_version
+            remote_db_version = 0
+            Net::HTTP.start(@remote_hostname) do |http|
+                resp = http.get(File.join(@remote_path, "db_version"))
+                resp.body.to_i
+            end
+        end
+
+        def update
+            check_local_dir
+            remote_db_version = get_remote_db_version
+            db_version = get_db_version
+            if db_version < remote_db_version
+                puts "Database update found!"
+                puts "Updating from version #{db_version} to version #{remote_db_version} (several GB to download - this can take a while)..."
+                download_all
+                set_db_version(remote_db_version)
+            end
+            puts "The database is up to date (version #{get_db_version})."
+        end
+
+        def download_all
+            download_file "stackoverflow_idx.db.gz"
+            gunzip_file   "stackoverflow_idx.db.gz"
+            download_file "stackoverflow.db.gz"
+            gunzip_file   "stackoverflow.db.gz"
+        end
+
+        def download_file(filename)
+            puts "Downloading #{filename}..."
+            Net::HTTP.start(@remote_hostname) do |http|
+                f = open(File.join(@dir_path, filename), "wb")
+                begin
+                    http.request_get(File.join(@remote_path, filename)) do |resp|
+                        resp.read_body do |segment|
+                            f.write(segment)
+                        end
+                    end
+                ensure
+                    f.close()
+                end
+            end
+        end
+
+        def gunzip_file(filename)
+            puts "Unpacking #{filename}..."
+            gz_file_path = File.join(@dir_path, filename)
+            z = Zlib::Inflate.new(16+Zlib::MAX_WBITS)
+
+            File.open(gz_file_path) do |f|
+                File.open(gz_file_path[0...-3], "w") do |w|
+                    f.each do |str|
+                        w << z.inflate(str)
+                    end
+                end
+            end
+            z.finish
+            z.close
+            File.delete(gz_file_path)
+        end
+
+        def directory_exists?(path)
+          return false if not File.exist?(path) or not File.directory?(path)
+          true
+        end
+
+        def file_exists?(path)
+          return false if not File.exist?(path) or not File.file?(path)
+          true
+        end
+
+    end
+
     class Formatter
         def questions_list(questions)
             nb = 1
@@ -126,15 +272,25 @@ module StackOverflow
         def run
             options = {}
             OptionParser.new do |opts|
-                opts.banner = "Usage: so [options] <search string> [<question_id>]"
+                opts.banner = "** Usage: so [options] <search string> [<question_id>]"
                 opts.on("-h", "--help", "Help") do |v|
-                    help(opts)
+                    help
+                    exit
+                end
+                opts.on("-u", "--update", "Update local database") do |v|
+                    DBUpdater.new.update
+                    exit
                 end
                 opts.on("-o", "--offline", "Offline mode") do |v|
                     options['offline'] = true
                 end
             end.parse!
 
+            if ARGV.length < 1
+                help
+                exit
+            end
+
             # last argument is integer when user is specifing a question_nb from the results
             question_nb = nil
             if ARGV[-1] =~ /^[0-9]+$/
@@ -142,20 +298,23 @@ module StackOverflow
             end
 
             search_string = ARGV.join(' ')
-
             search(search_string, question_nb, options)
         end
 
-        def help(opts)
-            puts opts.banner + "\n\n"
-            puts "\t<search_string>: Search Stack Overflow for a combination of words"
-            puts "\t<question_id>: (Optional) Display the question with this #id from the search results"
+        def help
+            puts "** Usage: so [options] <search string> [<question_id>]"
+            puts "          so --update\n\n"
+            puts "Arguments:"
+            puts "\t<search_string> : Search Stack Overflow for a combination of words"
+            puts "\t<question_id>   : (Optional) Display the question with this #id from the search results\n\n"
+            puts "Options:"
+            puts "\t-o --offline    : Query the local database instead of the online StackOverflow API (offline mode)"
+            puts "\t-u --update     : Download or update the local database of StackOverflow answers (7GB+)\n\n"
         end
 
         def search(search_string, question_nb, options)
             if options['offline']
-                api = API.new
-                #api = DB.new
+                api = DB.new
             else
                 api = API.new
             end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Xavier Antoviaque
@@ -16,18 +16,93 @@ cert_chain: []
 
 date: 2012-03-24 00:00:00 -04:00
 default_executable: 
-dependencies: []
-
-description: Query StackOverflow from the command line (offline/online modes)
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 6
+        - 5
+        version: 1.6.5
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: libxml-ruby
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 3
+        - 2
+        version: 2.3.2
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 5
+        version: 1.3.5
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: terminal-table
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 4
+        - 5
+        version: 1.4.5
+  type: :runtime
+  version_requirements: *id005
+description: Allows to query Stack Overflow's questions & answers from the command line. It can either be used in 'online' mode, where the StackOverflow API is queried, or offline, by downloading the latest dump released by StackOverflow.
 email: xavier@antoviaque.org
-executables: []
-
+executables: 
+- so
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
 - lib/stackoverflow.rb
+- bin/so
 has_rdoc: true
 homepage: https://github.com/antoviaque/stack-overflow-command-line
 licenses: []