birdwatcher 0.1.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8682e5d657e206d66a18fd2ee17cab1e0e696ceb
-  data.tar.gz: ae207377669987a05746239339a508e5119b1020
+  metadata.gz: 9967717a3d089165d8c2184b7028cf965d747277
+  data.tar.gz: 098b63db5bf37e20bb3a5d72a515c01b1dad42e7
 SHA512:
-  metadata.gz: ecc9517484aca7ed325d66ae7e2e1cf074a5230d6c00ab961a4656c352e4bfe5dae1201984d2ef991bcb504c5d3bcfeb7c4a3f861e19c001e4bb3461c1f42f74
-  data.tar.gz: c234083a65ba5f810768b9a4a833eadd275388d66cf371c7e33fb980829f47772f9acb2a5e496bff5e9a10fb829dca33a5698cf5a1caba4f1ebdb543b2b363cc
+  metadata.gz: 6bd0fb95954c3493cb7b3adce85a68f2b86a439b87fb81477e00ba7f0fce3b12b6c2e7cfb38b9744c19d5345d1a7748c0aaa11e138c0028981337fd167918688
+  data.tar.gz: fe6e8192f569912ab7c3e3e6d7e51496fd408c48a11b713d883d387e155a2154a637414153da14ccf243af1f949dc4969dbfdc46c1175e968827710becf2f468

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+
+## [0.3.1]
+### Added
+- New module `reporting/json` to export data from an SQL query to a file in  JSON format
+- New module `reporting/csv` to export data from an SQL query to a file in CSV format
+- This changelog
+
+### Changed
+- `posted_at` column added to `urls` for better and easier ordering
+
+### Fixed
+- make `status search` command case insensitive

data/db/migrations/016_add_posted_at_to_urls.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up do
+    alter_table(:urls) do
+      add_column :posted_at, DateTime
+      add_index :posted_at
+    end
+  end
+
+  down do
+    alter_table(:urls) do
+      drop_column :posted_at
+    end
+  end
+end

data/lib/birdwatcher/commands/status.rb

@@ -66,13 +66,13 @@ USAGE
       end
 
       def search_statuses
-        search_term = arguments[1..-1].join(" ")
+        search_term = arguments[1..-1].join(" ").downcase
         if search_term.empty?
           error("You must provide a search term")
           return false
         end
         statuses = current_workspace.statuses_dataset
-          .where("text LIKE ?", "%#{search_term}%")
+          .where("lower(text) LIKE ?", "%#{search_term}%")
           .order(Sequel.desc(:posted_at))
           .eager(:user)
           .limit(1000)

data/lib/birdwatcher/concerns/persistence.rb

@@ -61,7 +61,7 @@ module Birdwatcher
         if status.urls?
           status.urls.each do |url|
             expanded_url = Birdwatcher::Util.strip_control_characters(url.expanded_url.to_s)
-            db_url = current_workspace.urls_dataset.first(:url => expanded_url) || current_workspace.add_url(:url => expanded_url)
+            db_url = current_workspace.urls_dataset.first(:url => expanded_url) || current_workspace.add_url(:url => expanded_url, :posted_at => status.created_at)
             db_status.add_url(db_url)
           end
         end

data/lib/birdwatcher/concerns/util.rb

@@ -114,6 +114,25 @@ module Birdwatcher
         Birdwatcher::Util.excerpt(text, max_length, omission)
       end
 
+      # Create a more human readable representation of a number
+      #
+      # @param number [Numeric] number to make human readable
+      #
+      # @example
+      #     number_to_human_size(1024)
+      #     #=> "1KB"
+      #
+      #     number_to_human_size(2097152)
+      #     #=> "2MB"
+      #
+      #     number_to_human_size(3650722201)
+      #     #=> "3.4GB"
+      #
+      # @return [String] Human readable number
+      def number_to_human_size(number)
+        Birdwatcher::Util.number_to_human_size(number)
+      end
+
       # Suppress any potential output to STDOUT
       #
       # Used in cases where certain libraries or methods might output unwanted

data/lib/birdwatcher/modules/reporting/csv.rb

@@ -0,0 +1,71 @@
+module Birdwatcher
+  module Modules
+    module Reporting
+      class Csv < Birdwatcher::Module
+        self.meta = {
+          :name        => "CSV Exporter",
+          :description => "Export result from SQL query to a CSV file",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "QUERY" => {
+              :value       => nil,
+              :description => "SQL query to execute",
+              :required    => true
+            },
+            "HEADERS" => {
+              :value       => true,
+              :description => "Add CSV headers to the file",
+              :required    => false,
+              :boolean     => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The CSV exporter can write the results of an SQL query to a file in CSV format.
+
+#{'IMPORTANT:'.bold} The module does not limit the data returned from the query
+to the currently active workspace, the query will need to take that in to
+consideration if necessary.
+INFO
+        end
+
+        def run
+          result      = nil
+          rows        = nil
+          headers     = nil
+          csv         = nil
+          destination = option_setting("DEST")
+          task("Executing SQL query...") do
+            begin
+              result  = database[option_setting("QUERY")]
+              rows    = result.map { |r| r.to_hash.values }
+              headers = result.columns.map { |c| c.to_s }
+            rescue Sequel::DatabaseError => e
+              error("Syntax error: #{e.message}")
+              return false
+            end
+          end
+          task("Generating CSV...") do
+            csv = CSV.generate(:write_headers => option_setting("HEADERS"), :headers => headers) do |doc|
+              rows.each { |r| doc << r }
+            end
+          end
+          task("Writing #{pluralize(rows.count, 'row', 'rows')} to file...") do
+            File.open(destination, "w") do |f|
+              f.write(csv)
+            end
+          end
+          file_size = number_to_human_size(File.size(destination))
+          info("Wrote #{file_size.bold} to #{destination.bold}")
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/reporting/json.rb

@@ -0,0 +1,71 @@
+module Birdwatcher
+  module Modules
+    module Reporting
+      class Json < Birdwatcher::Module
+        self.meta = {
+          :name        => "JSON Exporter",
+          :description => "Export result from SQL query to a JSON file",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "QUERY" => {
+              :value       => nil,
+              :description => "SQL query to execute",
+              :required    => true
+            },
+            "PRETTY_FORMATTING" => {
+              :value       => false,
+              :description => "Output pretty formatted JSON",
+              :required    => false,
+              :boolean     => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The JSON exporter can write the results of an SQL query to a file in JSON format.
+
+#{'IMPORTANT:'.bold} The module does not limit the data returned from the query
+to the currently active workspace, the query will need to take that in to
+consideration if necessary.
+INFO
+        end
+
+        def run
+          result      = nil
+          rows        = nil
+          json        = nil
+          destination = option_setting("DEST")
+          task("Executing SQL query...") do
+            begin
+              result  = database[option_setting("QUERY")]
+              rows    = result.map { |r| r.to_hash }
+            rescue Sequel::DatabaseError => e
+              error("Syntax error: #{e.message}")
+              return false
+            end
+          end
+          task("Generating JSON...") do
+            if option_setting("PRETTY_FORMATTING")
+              json = JSON.pretty_generate(rows)
+            else
+              json = JSON.generate(rows)
+            end
+          end
+          task("Writing #{pluralize(rows.count, 'row', 'rows')} to file...") do
+            File.open(destination, "w") do |f|
+              f.write(json)
+            end
+          end
+          file_size = number_to_human_size(File.size(destination))
+          info("Wrote #{file_size.bold} to #{destination.bold}")
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/urls/crawl.rb

@@ -80,11 +80,11 @@ INFO
           if option_setting("RETRY_FAILED")
             urls = current_workspace.urls_dataset
               .where("crawled_at IS NULL or (crawled_at IS NOT NULL AND http_status IS NULL)")
-              .order(Sequel.desc(:created_at))
+              .order(Sequel.desc(:posted_at))
           else
             urls = current_workspace.urls_dataset
               .where(:crawled_at => nil)
-              .order(Sequel.desc(:created_at))
+              .order(Sequel.desc(:posted_at))
           end
           if urls.empty?
             error("There are currently no URLs in this workspace")

data/lib/birdwatcher/util.rb

@@ -1,5 +1,7 @@
 module Birdwatcher
   module Util
+    HUMAN_PREFIXES = %w(TB GB MB KB B).freeze
+
     def self.time_ago_in_words(time)
       return "a very very long time ago" if time.year < 1800
       secs = Time.now - time
@@ -61,6 +63,16 @@ module Birdwatcher
       text[0..max_length] + omission
     end
 
+    def self.number_to_human_size(number)
+      s = number.to_f
+      i = HUMAN_PREFIXES.length - 1
+      while s > 512 && i > 0
+        i -= 1
+        s /= 1024
+      end
+      ((s > 9 || s.modulo(1) < 0.1 ? "%d" : "%.1f") % s) + HUMAN_PREFIXES[i]
+    end
+
     def self.suppress_output(&block)
       original_stdout = $stdout
       $stdout = fake = StringIO.new

data/lib/birdwatcher/version.rb

@@ -1,3 +1,3 @@
 module Birdwatcher
-  VERSION = "0.1.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: birdwatcher
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Michael Henrikesn
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-14 00:00:00.000000000 Z
+date: 2016-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -296,6 +296,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -319,6 +320,7 @@ files:
 - db/migrations/013_create_influencers_users.rb
 - db/migrations/014_create_influencees.rb
 - db/migrations/015_create_influencees_users.rb
+- db/migrations/016_add_posted_at_to_urls.rb
 - exe/birdwatcher
 - lib/birdwatcher.rb
 - lib/birdwatcher/command.rb
@@ -353,6 +355,8 @@ files:
 - lib/birdwatcher/klout_client.rb
 - lib/birdwatcher/kml.rb
 - lib/birdwatcher/module.rb
+- lib/birdwatcher/modules/reporting/csv.rb
+- lib/birdwatcher/modules/reporting/json.rb
 - lib/birdwatcher/modules/statuses/kml.rb
 - lib/birdwatcher/modules/statuses/sentiment.rb
 - lib/birdwatcher/modules/statuses/word_cloud.rb