persistent-shell-history 0.0.3

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+*.gem
+.*swp
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in persistent-shel-history.gemspec
+gemspec

data/README

@@ -0,0 +1,34 @@
+This is a quick job, to have a local database, to collect _all_ commands
+from ~/.bash_history
+
+To couple with this, I have set a crontab job:
+  */30 * * * * /home/vbatts/bin/bash_history.rb
+
+
+and set in ~/.bashrc:
+  unset HISTFILESIZE
+  export HISTSIZE=10000
+  export HISTTIMEFORMAT="%F %T "
+  export HISTCONTROL="ignoreboth"
+
+
+== USAGE
+See the --help also,
+Usage: bash_history [options]
+        --inspect                    inspect the data
+    -h, --history FILE               use bash_history FILE instead of the default (~/.bash_history)
+    -d, --db FILE                    use database FILE instead of the default (~/.bash_history.db)
+    -l, --list                       list history
+        --fix                        fix times
+        --format FORMAT              specify a different strftime format. (default is "%F %T")
+    -f, --find PAT                   find a command with pattern PAT
+
+
+== LICENSE
+Copyright (c) 2012  Vincent Batts, Raleigh, NC, USA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler/gem_tasks'
+
+task :default => :build
+

data/TODOs

@@ -0,0 +1,17 @@
+* add a ~/.persistent-shell-history.yaml
+* move the db from gdbm to sqlite
+ - tables
+  ** hostname
+   -- id
+   -- name
+  ** command
+   -- id
+   -- value
+  ** record
+   -- id
+   -- timestamp
+   -- hostname_id
+   -- command_id
+* migration from current Marshal version need only to check the value, for if
+  it starts with "\x04\b{"
+

data/bin/bash_history.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+require 'persistent-shell-history/binary-history-store'
+require 'persistent-shell-history/old-history-store'
+require 'optparse'
+
+options = {}
+bh_options = {:archive_file => File.expand_path("~/.bash_history.db")}
+opts = OptionParser.new do |opts|
+  opts.on('--inspect','inspect the data') do |o|
+    options[:inspect] = o
+  end
+  opts.on('-h','--history FILE','use bash_history FILE instead of the default (~/.bash_history)') do |o|
+    bh_options[:file] = o
+  end
+  opts.on('-d','--db FILE','use database FILE instead of the default (~/.bash_history.db)') do |o|
+    bh_options[:archive_file] = o
+  end
+  opts.on('--migrate','check-for and migrate, only') do |o|
+    options[:migrate] = o
+  end
+  opts.on('-l','--list','list history') do |o|
+    options[:list] = o
+  end
+  opts.on('--format FORMAT','specify a different strftime format. (default is "%F %T")') do |o|
+    bh_options[:time_format] = o
+  end
+  opts.on('-f','--find PAT','find a command with pattern PAT') do |o|
+    options[:find] = o
+  end
+end
+
+begin
+  opts.parse!(ARGV)
+rescue => ex
+  puts ex
+  puts opts
+end
+
+def migrate(old_bashhistorystore)
+  require 'fileutils'
+  # migrate
+  temp_db = GDBM.new("#{old_bashhistorystore.archive}.#{$$}")
+  old_bashhistorystore.keys.each {|key|
+    h = old_bashhistorystore[key]
+    h[:time] = h[:time].map {|t| t.to_i }
+    temp_db[key] = Marshal.dump(h)
+  }
+  old_bashhistorystore.db.close()
+  temp_db.close()
+  ts = "#{Time.now.year}#{Time.now.month}#{Time.now.day}#{Time.now.hour}#{Time.now.min}"
+  old_filename = "#{old_bashhistorystore.archive}.old.#{ts}"
+  FileUtils.mv(old_bashhistorystore.archive, old_filename)
+  puts "archived [#{old_bashhistorystore.archive}] to [#{old_filename}] ..."
+  FileUtils.mv("#{old_bashhistorystore.archive}.#{$$}", old_bashhistorystore.archive)
+end
+
+# First check the database, for whether it is the old format,
+# if so, convert it, and reopen it.
+bh = Persistent::Shell::OldHistoryStore.new(bh_options)
+migrate(bh) if bh.is_oldformat?
+bh.db.close unless bh.db.closed?
+
+exit(0) if options[:migrate]
+
+# re-open the history storage
+bh = Persistent::Shell::BinaryHistoryStore.new(bh_options)
+
+# load the new bash_history into the database
+unless (options[:inspect] or options[:find] or options[:list])
+  bh.load()
+  bh.db.reorganize()
+end
+
+if options[:inspect]
+  p bh
+  #p "storing #{bh.keys.count} commands"
+end
+if options[:find]
+  bh.find(options[:find]).sort_by {|x| x[:time] }.each do |val|
+    puts bh.fmt(val)
+  end
+elsif options[:list]
+  bh.values_by_time.each do |val|
+    puts bh.fmt(val)
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history.rb

@@ -0,0 +1,9 @@
+require 'persistent-shell-history/version'
+require 'persistent-shell-history/history'
+
+module Persistent
+  module Shell
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history/abstract-history-store.rb

@@ -0,0 +1,17 @@
+
+module Persistent
+  module Shell
+
+    class AbstractHistoryStore
+      SCHEMA_VERSION = "1"
+
+      def commands; end
+      def db; end
+      def shema_match?; db.has_key? "schema_version" and db["schema_version"] == SCHEMA_VERSION; end
+      def shema_version; db["schema_version"] if db.has_key? "schema_version" ; end
+    end # class AbstractHistoryStore
+
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history/binary-history-store.rb

@@ -0,0 +1,129 @@
+require 'digest/md5'
+require 'gdbm'
+require 'yaml'
+
+require 'persistent-shell-history/abstract-history-store'
+require 'persistent-shell-history/history'
+require 'persistent-shell-history/command'
+
+if RUBY_VERSION >= "1.9" # assuming you're running Ruby ~1.9
+  Encoding.default_external = Encoding::UTF_8
+  Encoding.default_internal = Encoding::UTF_8
+end
+
+module Persistent
+  module Shell
+    class BinaryHistoryStore < AbstractHistoryStore
+      OPTIONS = {
+        :file          => File.expand_path("~/.bash_history"),
+        :archive_file  => File.expand_path("~/.bash_history.db"),
+        :time_format   => "%F %T",
+      }
+
+      def initialize(opts = {})
+        @options = OPTIONS.merge(opts)
+      end
+
+      def archive; @options[:archive_file]; end
+      def archive=(arg); @options[:archive_file] = arg; end
+
+      def hist_file; @options[:file]; end
+      def hist_file=(arg); @options[:file] = arg; end
+
+      def time_format; @options[:time_format]; end
+      def time_format=(tf); @options[:time_format] = tf; end
+      def db
+        @db ||= GDBM.new(@options[:archive_file])
+      end
+      def [](key); _ml(db[key]); end
+      def keys; db.keys; end
+      def values; db.map {|k,v| _ml(v) }; end
+      def values_by_time
+        return db.map {|k,v|
+          data = _ml(v)
+          data[:time].map {|t|
+            data.merge(:time => t)
+          }
+        }.flatten.sort_by {|x|
+          x[:time]
+        }
+      end
+      def commands; values.map {|v| v[:cmd] }; end
+      def _md(data); Marshal.dump(data); end
+      def _ml(data); Marshal.load(data); end
+      def _md5(data); Digest::MD5.hexdigest(data); end
+
+      # display a formatted time commend
+      def fmt(cmd); " %s %s" % [Time.at(cmd[:time]).strftime(@options[:time_format]), cmd[:cmd]]; end
+
+      def find(pat)
+        return values.select {|v|
+          v if v[:cmd] =~ /#{pat}/
+        }.map {|v|
+          v[:time].map {|t|
+            v.merge(:time => t)
+          }
+        }.flatten
+      end
+
+      def load(filename = @options[:file])
+        open(filename) do |f|
+          f.each_line do |line|
+            if line =~ /^#(.*)$/
+              l = f.readline.chomp
+              key = _md5(l)
+              if db.has_key?(key)
+                times = _ml(db[key])[:time]
+                if times.kind_of? Array
+                  times.push($1.to_i)
+                else
+                  times = [times]
+                end
+                db[key] = _md({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _md({:cmd => l, :time => [$1.to_i] })
+              end
+            else
+              key = _md5(line.chomp)
+              if db.has_key?(key)
+                times = _ml(db[key])[:time]
+                if times.kind_of? Array
+                  times.push($1.to_i)
+                else
+                  times = [times]
+                end
+                db[key] = _md({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _md({:cmd => line.chomp, :time => [0] })
+              end
+            end
+          end
+        end
+      end
+
+      # returns a Persistent::Shell::History object from the current GDBM database.
+      # intended for marshalling to other history-stores
+      def to_history
+        history = History.new
+        values.each do |value|
+          value[:time].each do |t|
+            history << Command.new(value[:cmd], t.to_i)
+          end
+        end
+        return history
+      end
+
+      # create an output that looks like a regular ~/.bash_history file
+      def render(file)
+        File.open(file,'w+') do |f|
+          values.each do |v|
+            f.write("#" + v[:time].to_i.to_s + "\n") if v[:time] and not (v[:time].to_i == 0)
+            f.write(v[:cmd] + "\n")
+          end
+        end
+      end
+
+    end # class BinaryHistoryStore
+  end # Shell
+end # Persistent
+

data/lib/persistent-shell-history/command.rb

@@ -0,0 +1,21 @@
+
+require 'digest/md5'
+require 'json'
+
+module Persistent
+  module Shell
+    class Command < Struct.new(:cmd, :time)
+      def md5
+        Digest::MD5.hexdigest(cmd)
+      end
+      def to_h
+        { :cmd => cmd, :time => time, }
+      end
+      def to_json(*a)
+        to_h.to_json(*a)
+      end
+    end
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history/history.rb

@@ -0,0 +1,43 @@
+
+require 'persistent-shell-history/command'
+require 'json'
+
+module Persistent
+  module Shell
+    # Abstract storage for command history
+    class History
+      def initialize()
+        @cmds = Array.new
+      end
+      def commands; @cmds; end
+      def commands=(cmds); @cmds = cmds; end
+      def <<(arg); @cmds << arg; end
+      def to_a; commands.map {|c| c.to_h }; end
+      def to_json(*a); commands.to_json(*a); end
+    end
+    class BashHistory < History
+      def initialize(filename = '~/.bash_history')
+        @filename = File.expand_path(filename)
+      end
+      def commands; (@cmds.nil? or @cmds.empty?) ? (@cmds = parse) : @cmds; end
+      def file; @filename; end
+      def file=(filename); @filename = File.expand_path(filename); end
+      def parse(filename = @filename)
+        cmds = Array.new
+        open(filename) do |f|
+          f.each_line do |line|
+            if line =~ /^#(.*)$/
+              l = f.readline.chomp
+              cmds << Command.new(l, $1)
+            else
+              cmds << Command.new(line, "0")
+            end
+          end
+        end
+        return cmds
+      end
+    end
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history/old-history-store.rb

@@ -0,0 +1,141 @@
+
+require 'digest/md5'
+require 'gdbm'
+require 'yaml'
+
+require 'persistent-shell-history/abstract-history-store'
+require 'persistent-shell-history/history'
+require 'persistent-shell-history/command'
+
+module Persistent
+  module Shell
+    class OldHistoryStore < AbstractHistoryStore
+      OPTIONS = {
+        :file          => File.expand_path("~/.bash_history"),
+        :archive_file  => File.expand_path("~/.bash_history.db"),
+        :time_format   => "%F %T",
+      }
+
+      def initialize(opts = {})
+        @options = OPTIONS.merge(opts)
+      end
+
+      def archive; @options[:archive_file]; end
+      def archive=(arg); @options[:archive_file] = arg; end
+
+      def hist_file; @options[:file]; end
+      def hist_file=(arg); @options[:file] = arg; end
+
+      # check the archive, whether it's the old format.
+      # If so, it needs to be converted to the BinaryHistoryStore
+      def is_oldformat?
+        db.keys[0..5].each {|key|
+          begin
+            YAML.load(db[key])
+          rescue Psych::SyntaxError => ex
+            return false
+          rescue => ex
+            return false
+          end
+        }
+        return true
+      end
+
+      def time_format; @options[:time_format]; end
+      def time_format=(tf); @options[:time_format] = tf; end
+      def db
+        @db ||= GDBM.new(@options[:archive_file])
+      end
+      def [](key); _yl(db[key]); end
+      def keys; db.keys; end
+      def keys_to_i; keys.map {|i| i.to_i }; end
+      def values; db.map {|k,v| _yl(v) }; end
+      def values_by_time
+        return db.map {|k,v|
+          data = _yl(v)
+          data[:time].map {|t|
+            data.merge(:time => t)
+          }
+        }.flatten.sort_by {|x|
+          x[:time]
+        }
+      end
+      def commands; values.map {|v| v[:cmd] }; end
+      def _yd(data); YAML.dump(data); end
+      def _yl(data); YAML.load(data); end
+      def _md5(data); Digest::MD5.hexdigest(data); end
+
+      # display a formatted time commend
+      def fmt(cmd); " %s %s" % [cmd[:time].strftime(@options[:time_format]), cmd[:cmd]]; end
+    
+      def find(pat)
+        return values.select {|v|
+          v if v[:cmd] =~ /#{pat}/
+        }.map {|v|
+          v[:time].map {|t|
+            v.merge(:time => t)
+          }
+        }.flatten
+      end
+    
+      def load(filename = @options[:file])
+        open(filename) do |f|
+          f.each_line do |line|
+            if line =~ /^#(.*)$/
+              l = f.readline.chomp
+              key = _md5(l)
+              if db.has_key?(key)
+                times = _yl(db[key])[:time]
+                if times.kind_of? Array
+                  times.push(Time.at($1.to_i))
+                else
+                  times = [times]
+                end
+                db[key] = _yd({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _yd({:cmd => l, :time => [Time.at($1.to_i)] })
+              end
+            else
+              key = _md5(line.chomp)
+              if db.has_key?(key)
+                times = _yl(db[key])[:time]
+                if times.kind_of? Array
+                  times.push(Time.at($1.to_i))
+                else
+                  times = [times]
+                end
+                db[key] = _yd({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _yd({:cmd => line.chomp, :time => [Time.at(0)] })
+              end
+            end
+          end
+        end
+      end
+
+      # returns a Persistent::Shell::History object from the current GDBM database.
+      # intended for marshalling to other history-stores
+      def to_history
+        history = History.new
+        values.each do |value|
+          value[:time].each do |t|
+            history << Command.new(value[:cmd], t.to_i)
+          end
+        end
+        return history
+      end
+
+      # create an output that looks like a regular ~/.bash_history file
+      def render(file)
+        File.open(file,'w+') do |f|
+          values.each do |v|
+            f.write("#" + v[:time].to_i.to_s + "\n") if v[:time] and not (v[:time].to_i == 0)
+            f.write(v[:cmd] + "\n")
+          end
+        end
+      end
+    end # class DataStore
+  end # module Shell
+end # module Persistent
+
+# vim: set sts=2 sw=2 et ai:

data/lib/persistent-shell-history/version.rb

@@ -0,0 +1,7 @@
+module Persistent
+  module Shell
+    VERSION = "0.0.3"
+  end
+end
+
+# vim: set sts=2 sw=2 et ai:

data/persistent-shell-history.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "persistent-shell-history/version"
+
+Gem::Specification.new do |s|
+  s.name        = "persistent-shell-history"
+  s.version     = Persistent::Shell::VERSION
+  s.authors     = ["Vincent Batts"]
+  s.email       = ["vbatts@hashbangbash.com"]
+  s.homepage    = ""
+  s.summary     = %q{bash shell history collection}
+  s.description = %q{
+This is a quick job, to have a local database, to collect _all_ commands
+from ~/.bash_history
+See README for other implementation helpers.
+}
+
+  s.rubyforge_project = "persistent-shell-history"
+  s.add_dependency("json")
+  s.add_dependency("gdbm")
+  s.add_dependency("ffi")
+
+  s.add_dependency("activerecord")
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: persistent-shell-history
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+  prerelease: 
+platform: ruby
+authors:
+- Vincent Batts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &11562800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11562800
+- !ruby/object:Gem::Dependency
+  name: gdbm
+  requirement: &11562380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11562380
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: &11561960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11561960
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &11561540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *11561540
+description: ! '
+
+  This is a quick job, to have a local database, to collect _all_ commands
+
+  from ~/.bash_history
+
+  See README for other implementation helpers.
+
+'
+email:
+- vbatts@hashbangbash.com
+executables:
+- bash_history.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- TODOs
+- bin/bash_history.rb
+- lib/persistent-shell-history.rb
+- lib/persistent-shell-history/abstract-history-store.rb
+- lib/persistent-shell-history/binary-history-store.rb
+- lib/persistent-shell-history/command.rb
+- lib/persistent-shell-history/history.rb
+- lib/persistent-shell-history/old-history-store.rb
+- lib/persistent-shell-history/version.rb
+- persistent-shell-history.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: persistent-shell-history
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: bash shell history collection
+test_files: []
+has_rdoc: