RubyGems - pipio - Versions diffs - 0.0.1 - Mend

pipio 0.0.1

Files changed (66) hide show

checksums.yaml +7 -0
data/.gitignore +27 -0
data/.rspec +2 -0
data/.simplecov +5 -0
data/.travis.yml +12 -0
data/Gemfile +3 -0
data/LICENSE +20 -0
data/NEWS.md +10 -0
data/README.md +88 -0
data/Rakefile +13 -0
data/lib/pipio.rb +34 -0
data/lib/pipio/alias_registry.rb +26 -0
data/lib/pipio/chat.rb +39 -0
data/lib/pipio/cleaners/html_cleaner.rb +95 -0
data/lib/pipio/cleaners/text_cleaner.rb +15 -0
data/lib/pipio/file_reader.rb +29 -0
data/lib/pipio/message_creators/auto_or_xml_message_creator.rb +25 -0
data/lib/pipio/message_creators/event_message_creator.rb +47 -0
data/lib/pipio/message_creators/status_message_creator.rb +19 -0
data/lib/pipio/messages/auto_reply_message.rb +7 -0
data/lib/pipio/messages/event.rb +67 -0
data/lib/pipio/messages/message.rb +23 -0
data/lib/pipio/messages/status_message.rb +26 -0
data/lib/pipio/messages/xml_message.rb +43 -0
data/lib/pipio/metadata.rb +34 -0
data/lib/pipio/metadata_parser.rb +55 -0
data/lib/pipio/parser_factory.rb +32 -0
data/lib/pipio/parsers/basic_parser.rb +83 -0
data/lib/pipio/parsers/html_log_parser.rb +22 -0
data/lib/pipio/parsers/null_parser.rb +9 -0
data/lib/pipio/parsers/text_log_parser.rb +21 -0
data/lib/pipio/tag_balancer.rb +163 -0
data/lib/pipio/time_parser.rb +36 -0
data/lib/pipio/version.rb +3 -0
data/pipio.gemspec +27 -0
data/spec/pipio/alias_registry_spec.rb +37 -0
data/spec/pipio/chat_spec.rb +66 -0
data/spec/pipio/cleaners/html_cleaner_spec.rb +102 -0
data/spec/pipio/cleaners/text_cleaner_spec.rb +29 -0
data/spec/pipio/file_reader_spec.rb +130 -0
data/spec/pipio/messages/auto_reply_message_spec.rb +40 -0
data/spec/pipio/messages/event_spec.rb +41 -0
data/spec/pipio/messages/status_message_spec.rb +43 -0
data/spec/pipio/messages/xml_message_spec.rb +55 -0
data/spec/pipio/metadata_parser_spec.rb +81 -0
data/spec/pipio/metadata_spec.rb +72 -0
data/spec/pipio/parser_factory_spec.rb +31 -0
data/spec/pipio/parsers/html_log_parser_spec.rb +160 -0
data/spec/pipio/parsers/null_parser_spec.rb +13 -0
data/spec/pipio/parsers/text_log_parser_spec.rb +37 -0
data/spec/pipio/tag_balancer_spec.rb +16 -0
data/spec/pipio/time_parser_spec.rb +66 -0
data/spec/pipio_spec.rb +63 -0
data/spec/spec_helper.rb +18 -0
data/spec/support/chat_builder.rb +29 -0
data/spec/support/chat_builder_helpers.rb +41 -0
data/spec/support/file_builder.rb +22 -0
data/spec/support/html_chat_builder.rb +67 -0
data/spec/support/logfiles/2006-12-21.223606.txt +3 -0
data/spec/support/logfiles/2008-01-15.071445-0500PST.htm +5 -0
data/spec/support/logfiles/2008-01-15.071445-0500PST.html +5 -0
data/spec/support/text_chat_builder.rb +21 -0
data/spec/test-output/README.md +1 -0
data/spec/test-output/html_log_output.xml +6 -0
data/spec/test-output/text_log_output.xml +4 -0
metadata +193 -0

data/lib/pipio/parsers/html_log_parser.rb ADDED

@@ -0,0 +1,22 @@
+module Pipio
+  class HtmlLogParser
+    TIMESTAMP_REGEX = /\((?<timestamp>(?:\d{4}-\d{2}-\d{2} )?\d{1,2}:\d{1,2}:\d{1,2}(?: [AP]M)?)\)/
+    def initialize(source_file_path, user_aliases)
+      # @line_regex matches a line in an HTML log file other than the first.
+      line_regex = /#{TIMESTAMP_REGEX} ?<b>(?<sn_or_alias>.+?) ?(?<auto_reply>&lt;AUTO-REPLY&gt;)?:?<\/b> ?(?<body>.+)<br ?\/>/o
+      # @line_regex_status matches a status or event line.
+      line_regex_status = /#{TIMESTAMP_REGEX} ?<b> (?<body>.+)<\/b><br ?\/>/o
+      cleaner = Cleaners::HtmlCleaner
+      @parser = BasicParser.new(source_file_path, user_aliases, line_regex,
+        line_regex_status, cleaner)
+    end
+    def parse
+      @parser.parse
+    end
+  end
+end

data/lib/pipio/parsers/null_parser.rb ADDED

@@ -0,0 +1,9 @@
+module Pipio
+  class NullParser
+    def initialize(logfile_path, aliases)
+    end
+    def parse
+    end
+  end
+end

data/lib/pipio/parsers/text_log_parser.rb ADDED

@@ -0,0 +1,21 @@
+module Pipio
+  class TextLogParser
+    TIMESTAMP_REGEX = '\((?<timestamp>\d{1,2}:\d{1,2}:\d{1,2})\)'
+    def initialize(source_file_path, user_aliases)
+      # @line_regex matches a line in a text log file other than the first.
+      line_regex = /#{TIMESTAMP_REGEX} (?<sn_or_alias>.*?) ?(?<auto_reply><AUTO-REPLY>)?: (?<body>.*)/o
+      # @line_regex_status matches a status or event line.
+      line_regex_status = /#{TIMESTAMP_REGEX} (?<body>[^:]+)/o
+      cleaner = Cleaners::TextCleaner
+      @parser = BasicParser.new(source_file_path, user_aliases, line_regex,
+        line_regex_status, cleaner)
+    end
+    def parse
+      @parser.parse
+    end
+  end
+end

data/lib/pipio/tag_balancer.rb ADDED

@@ -0,0 +1,163 @@
+module Pipio
+  # Balances tags of string using a modified stack. Returns a balanced
+  # string, but also affects the text passed into it!
+  # Use text = balance_tags(text).
+  # From Wordpress's formatting.php; rewritten in Ruby by Gabe
+  # Berke-Williams, 2009.
+  # Author:: Leonard Lin <leonard@acm.org>
+  # License:: GPL v2.0
+  # Copyright:: November 4, 2001
+  class TagBalancer
+    def initialize(text)
+      @text = text
+      @tagstack  = []
+      @stacksize = 0
+      @tagqueue  = ''
+      # Known single-entity/self-closing tags
+      @self_closing_tags = %w(br hr img input meta)
+      # Tags that can be immediately nested within themselves
+      @nestable_tags = %w(blockquote div span font)
+      # 1: tagname, with possible leading "/"
+      # 2: attributes
+      @tag_regex = /<(\/?\w*)\s*([^>]*)>/
+    end
+    def balance
+      text    = @text.dup
+      newtext = ''
+      @tagstack  = []
+      @stacksize = 0
+      @tagqueue  = ''
+      # WP bug fix for comments - in case you REALLY meant to type '< !--'
+      text.gsub!('< !--', '<    !--')
+      # WP bug fix for LOVE <3 (and other situations with '<' before a number)
+      text.gsub!(/<([0-9]{1})/, '&lt;\1')
+      while ( pos = (text =~ @tag_regex) )
+        newtext << @tagqueue
+        tag        = $1.downcase
+        attributes = $2
+        matchlen   = $~[0].size
+        # clear the shifter
+        @tagqueue = ''
+        # Pop or Push
+        if end_tag?(tag)
+          tag.slice!(0,1)
+          if too_many_closing_tags?
+            tag = ''
+            #or close to be safe: tag = '/' << tag
+          elsif closing_tag?(tag)
+            # if stacktop value == tag close value then pop
+            tag = "</#{tag}>" # Close Tag
+            @tagstack.pop
+            @stacksize -= 1
+          else
+            # closing tag not at top, search for it
+            (@stacksize-1).downto(0) do |j|
+              if @tagstack[j] == tag
+                # add tag to tagqueue
+                ss = @stacksize - 1
+                ss.downto(j) do |k|
+                  @tagqueue << "</#{@tagstack.pop}>"
+                  @stacksize -= 1
+                end
+                break
+              end
+            end
+            tag = ''
+          end
+        else
+          # Begin Tag
+          # Tag Cleaning
+          if self_closing_attributes?(attributes) || empty_tag?(tag)
+          elsif self_closing_tag?(tag)
+            # ElseIf: it's a known single-entity tag but it doesn't close itself, do so
+            attributes << '/'
+          else
+            # Push the tag onto the stack
+            # If the top of the stack is the same as the tag we want to push, close previous tag
+            if (@stacksize > 0 &&
+                ! nestable?(tag) &&
+                @tagstack[@stacksize - 1] == tag)
+              @tagqueue = "</#{@tagstack.pop}>"
+              @stacksize -= 1
+            end
+            @tagstack.push(tag)
+            @stacksize += 1
+          end
+          # Attributes
+          if attributes != ''
+            attributes = ' ' + attributes
+          end
+          tag = "<#{tag}#{attributes}>"
+          #If already queuing a close tag, then put this tag on, too
+          if @tagqueue
+            @tagqueue << tag
+            tag = ''
+          end
+        end
+        newtext << text[0,pos] << tag
+        text = text[pos+matchlen, text.length - (pos+matchlen)]
+      end
+      # Clear Tag Queue
+      newtext << @tagqueue
+      # Add Remaining text
+      newtext << text
+      # Empty Stack
+      @tagstack.reverse_each do |t|
+        newtext << "</#{t}>" # Add remaining tags to close
+      end
+      # WP fix for the bug with HTML comments
+      newtext.gsub!("< !--", "<!--")
+      newtext.gsub!("<    !--", "< !--")
+      newtext
+    end
+    private
+    def end_tag?(string)
+      string[0,1] == "/"
+    end
+    def closing_tag?(tag)
+      @tagstack[@stacksize - 1] == tag
+    end
+    def too_many_closing_tags?
+      @stacksize <= 0
+    end
+    def self_closing_attributes?(attributes)
+      attributes[-1,1] == '/'
+    end
+    def empty_tag?(tag)
+      tag == ''
+    end
+    def self_closing_tag?(tag)
+      @self_closing_tags.include?(tag)
+    end
+    def nestable?(tag)
+      @nestable_tags.include?(tag)
+    end
+  end
+end

data/lib/pipio/time_parser.rb ADDED

@@ -0,0 +1,36 @@
+module Pipio
+  class TimeParser
+    NO_DATE = /\A\d{1,2}:\d{1,2}:\d{1,2}(?: [AP]M)?\Z/
+    # 01/22/2008 03:01:45 PM
+    UNPARSEABLE_BY_DATETIME_PARSE = '%m/%d/%Y %I:%M:%S %P'
+    def initialize(year, month, day)
+      @fallback_date_string = "#{year}-#{month}-#{day}"
+    end
+    def parse(timestamp)
+      if timestamp
+        if has_no_date?(timestamp)
+          parse_with_date(@fallback_date_string + " " + timestamp)
+        else
+          parse_with_date(timestamp)
+        end
+      end
+    end
+    private
+    def parse_with_date(timestamp)
+      begin
+        Time.parse(timestamp)
+      rescue ArgumentError
+        Time.strptime(timestamp, UNPARSEABLE_BY_DATETIME_PARSE)
+      end
+    end
+    def has_no_date?(timestamp)
+      timestamp.strip =~ NO_DATE
+    end
+  end
+end

data/lib/pipio/version.rb ADDED

@@ -0,0 +1,3 @@
+module Pipio
+  VERSION = "0.0.1"
+end

data/pipio.gemspec ADDED

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "pipio/version"
+Gem::Specification.new do |spec|
+  spec.name          = "pipio"
+  spec.version       = Pipio::VERSION
+  spec.authors       = ["Gabe Berke-Williams"]
+  spec.email         = "gabe@thoughtbot.com"
+  spec.description   = "A fast, easy way to parse Pidgin (gaim) logs"
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/gabebw/pipio"
+  spec.license       = "MIT"
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = Gem::Requirement.new(">= 1.9.2")
+  spec.add_development_dependency("mocha")
+  spec.add_development_dependency("rspec", "~> 3.0")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("simplecov")
+end

data/spec/pipio/alias_registry_spec.rb ADDED

@@ -0,0 +1,37 @@
+describe Pipio::AliasRegistry do
+  it 'keeps track of aliases' do
+    alias_registry['My Cool Alias'] = 'screen_name88'
+    expect(alias_registry['My Cool Alias']).to eq('screen_name88')
+  end
+  it 'finds aliases even when they are queried with an action' do
+    alias_registry['My Cool Alias'] = 'screen_name88'
+    expect(alias_registry['***My Cool Alias']).to eq('screen_name88')
+  end
+  it 'downcases screen names' do
+    alias_registry['alias'] = 'UPCASE'
+    expect(alias_registry['alias']).to eq('upcase')
+  end
+  it 'removes space from screen names' do
+    alias_registry['alias'] = 'a space'
+    expect(alias_registry['alias']).to eq('aspace')
+  end
+  it 'takes a default' do
+    alias_registry = Pipio::AliasRegistry.new('default_name')
+    expect(alias_registry['alias']).to eq('default_name')
+  end
+  it 'normalizes the default' do
+    alias_registry = Pipio::AliasRegistry.new('DEFAULT NAME')
+    expect(alias_registry['alias']).to eq('defaultname')
+  end
+  def alias_registry
+    @alias_registry ||= Pipio::AliasRegistry.new('default')
+  end
+end

data/spec/pipio/chat_spec.rb ADDED

@@ -0,0 +1,66 @@
+describe Pipio::Chat do
+  describe '#to_s' do
+    it 'converts all Messages to strings and joins them' do
+      chat = Pipio::Chat.new([:a, 1, 3], metadata)
+      expect(chat.to_s).to eq("a\n1\n3")
+    end
+  end
+  describe '#messages' do
+    it 'returns all messages' do
+      chat = Pipio::Chat.new(%w(a b c), metadata)
+      expect(chat.messages).to eq %w(a b c)
+    end
+  end
+  it 'is enumerable' do
+    chat = Pipio::Chat.new(%w(a b c), metadata)
+    expect(chat.map(&:upcase)).to eq(%w(A B C))
+  end
+  describe '#their_screen_name' do
+    it 'is the screen name of the other person in the chat' do
+      chat = Pipio::Chat.new([], metadata(their_screen_name: 'them'))
+      expect(chat.their_screen_name).to eq('them')
+    end
+  end
+  describe '#my_screen_name' do
+    it 'is my screen name' do
+      chat = Pipio::Chat.new([], metadata(my_screen_name: 'me'))
+      expect(chat.my_screen_name).to eq('me')
+    end
+  end
+  describe '#start_time_xmlschema' do
+    it 'is the start time of the chat in xmlschema format' do
+      time = Time.now
+      chat = Pipio::Chat.new([], metadata(start_time: time))
+      expect(chat.start_time_xmlschema).to eq(time.xmlschema)
+    end
+  end
+  describe '#service' do
+    it 'is the chat service' do
+      chat = Pipio::Chat.new([], metadata(service: 'icq'))
+      expect(chat.service).to eq('icq')
+    end
+  end
+  def metadata(options = {})
+    data = {
+      my_screen_name: 'me',
+      their_screen_name: 'them',
+      start_time: Time.now,
+      service: 'aim'
+    }.merge(options)
+    Pipio::Metadata.new(data)
+  end
+end

data/spec/pipio/cleaners/html_cleaner_spec.rb ADDED

@@ -0,0 +1,102 @@
+describe Pipio::Cleaners::HtmlCleaner, ".clean" do
+  it "removes html, body, and font tags" do
+    clean_text = 'clean'
+    dirty_text = %{<html><body><font color="red">#{clean_text}</font></body></html>}
+    expect(clean(dirty_text)).to eq(clean_text)
+  end
+  it "removes those weird <FONT HSPACE> tags" do
+    clean_text = 'clean'
+    dirty_text = "&lt;/FONT HSPACE='2'>#{clean_text}"
+    expect(clean(dirty_text)).to eq(clean_text)
+  end
+  it 'removes \r' do
+    clean_text = 'clean'
+    dirty_text = [clean_text, clean_text, clean_text].join("\r")
+    expect(clean(dirty_text)).to eq(clean_text * 3)
+  end
+  it "removes empty lines" do
+    clean_text = 'clean'
+    dirty_text = "#{clean_text}\n\n"
+    expect(clean(dirty_text)).to eq(clean_text)
+  end
+  it "replaces newlines with <br/>" do
+    clean_text = "<br/>clean"
+    dirty_text = "\nclean"
+    expect(clean(dirty_text)).to eq(clean_text)
+  end
+  it "removes empty links" do
+    clean_text = 'clean' * 2
+    dirty_text = '<a href="awesomelink">   </a>clean' +
+      "<a href='awesomelink'></a>clean"
+    expect(clean(dirty_text)).to eq(clean_text)
+  end
+  describe "with <span>s" do
+    it "removes font-family" do
+      clean_text = 'clean'
+      dirty_text = %Q{<span style='font-family: Helvetica;'>#{clean_text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "removes font-size" do
+      clean_text = 'clean'
+      dirty_text = %Q{<span style="font-size: 6;">#{clean_text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "removes background" do
+      clean_text = 'clean'
+      dirty_text = %Q{<span style="background: #00afaf;">#{clean_text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "removes color=#00000" do
+      clean_text = 'clean'
+      dirty_text = %Q{<span style="color: #000000;">#{clean_text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "does not remove color that is not #00000" do
+      dirty_text = %Q{<span style="color: #01ABcdef;">whatever</span>}
+      expect(clean(dirty_text)).to eq(dirty_text)
+    end
+    it "removes improperly-formatted colors" do
+      clean_text = 'clean'
+      dirty_text = %Q{<span style="color: #0;">#{clean_text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "replaces <em> with italic font-style" do
+      text = 'whatever'
+      dirty_text = "<em>#{text}</em>"
+      clean_text = %Q{<span style="font-style: italic;">#{text}</span>}
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+    it "does not modify clean text" do
+      expect(clean('clean')).to eq('clean')
+    end
+    # This implicitly tests a lot of other things, but they've been tested
+    # before this too.
+    it "removes a trailing space after style declaration and replaces double quotes" do
+      dirty_span_open = "<span style='color: #afaf00; font-size: 14pt; font-weight: bold; '>"
+      # Replaced double quotes, removed space before ">"
+      clean_span_open = '<span style="color: #afaf00;">'
+      text = 'whatever'
+      dirty_text = "#{dirty_span_open}#{text}</span>"
+      clean_text = "#{clean_span_open}#{text}</span>"
+      expect(clean(dirty_text)).to eq(clean_text)
+    end
+  end
+  def clean(line)
+    Pipio::Cleaners::HtmlCleaner.clean(line)
+  end
+end