mess 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 70d63987314f20d83db49e145f32fb23e5bb1b20608d343672c474f3aea128e4
+  data.tar.gz: 3883e201fb6e57978cd55215a9b4951e8c0133d441bd4a9514f4e2cb3f7410df
+SHA512:
+  metadata.gz: 98019de61d628b344ab85c05d81e5db2aecbb63cc101e8164cbec8395f871d31b1ac5fa4d56ad859d662d31698747a51f874c6e38a1f580b85198ce537ed90be
+  data.tar.gz: '09922eb1f81a2030e9a1fdfc1e8224b3886ad0048281048b4deb62a078d604df59e456eb5f6cb45f9212aed782e5b1275b449bb658f1c3640adc01c29a1f4a81'

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in mess.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/Gemfile.lock

@@ -0,0 +1,21 @@
+PATH
+  remote: .
+  specs:
+    mess (0.0.0)
+      slop (~> 4.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (12.3.3)
+    slop (4.8.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mess!
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 patztablook22
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# mess
+
+Ruby tool for analysis of native Facebook Messenger Archives in JSON format. Output is universal (also JSON) and can thus be used by table processors (like MS Excell / LibreOffice Calc) and more!
+
+## Installation
+Ruby is required. You can download mess as a Ruby Gem:
+```bash
+gem install mess
+```
+Alternatively, after cloning this repo, run:
+```bash
+bundle exec rake install
+```
+
+## Facebook archive
+provide the program with **JSON** (!) Facebook archive _(as opposed to HTML format)_
+
+## Examples
+Call `main` either directly or via link.
+```bash
+
+# print help
+main --help 
+
+# guided analysis of given archive
+main --facebook my/facebook/archive
+
+# analysis specifying chat and plots
+main --facebook my/facebook/archive --chat 3 --plot asdf
+
+# list all available chats and plots for given archive
+main --facebook my/facebook/archive --list --chat --plot
+
+```
+
+## Contributing
+Mess provides class `Mess::Plot` to be inherited for custom plots; \
+quick example (for more, see `mess/plots` dir):
+```ruby
+module Mess
+  class MyPlot < Plot
+
+    # this will make MyPlot "available"
+    describe "my plot for magic analysis"
+  
+    def initialize chat     # constructor
+      # blah blah           # whatever is needed
+      # more code           # should be only preparation
+    end
+  
+    def push msg            # being called for every single message chronologically
+      if msg["sender_name"].include? "my"
+        # do stuff
+     else
+        # don't etc.
+      end
+    end
+  
+    def review              # called after all messages have been push-ed
+      # body                # perhaps reformat the data and so on
+    end
+  
+  end
+end 
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mess"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/mess

@@ -0,0 +1,232 @@
+#! /usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'io/console'
+require 'slop'
+require_relative '../lib/mess'
+
+Signal.trap('INT') { exit }
+
+$listing = Array.new
+
+begin
+  opts = Slop.parse do |o|
+
+    o.banner = "usage: #{$0} [options] target"
+
+    o.on '--help', 'print help' do
+      puts o
+      exit
+    end
+
+    o.on '--version', 'print version' do
+      puts Mess::VERSION
+      exit
+    end
+
+    o.string '--output', '-o', 'output file', default: 'mess_out.txt'
+
+    o.bool '--archived', '-a', 'archived threads too', default: false
+
+    o.bool '--list', '-l', 'listing mode'
+
+    o.array '--chat', '-c', 'chat specification' do
+      $listing << :chats
+    end
+
+    o.array '--plot', '-p', 'plot specification' do
+      $listing << :plots
+    end
+
+  end
+rescue Slop::MissingArgument => e
+  p e.message
+rescue Slop::UnknownOption => e
+  puts e
+  exit 1
+end
+
+target = opts&.arguments || Array.new
+target << '.' if target.none?
+
+if target.one?
+  target = target.first
+else
+  $stderr.puts 'too many targets'
+  exit 1
+end
+
+def list_chats tree
+  puts "chats"
+  tree.chats.each_with_index do |chat, index|
+    index += 1
+    index  = index.to_s.rjust(5)
+    puts "#{index} | #{chat.title}"
+  end
+end
+
+def list_plots plots
+  puts "plots"
+  index = 1
+  plots.available.each_pair do |name, desc|
+    puts "#{index.to_s.rjust(5)} | #{name.ljust(15)} -> #{desc}"
+    index += 1
+  end
+end
+
+# listing mode
+if opts.list?
+
+  # chats
+  if $listing.include? :chats
+    begin
+      tree = Mess::Tree.new(target)
+      puts
+      list_chats tree
+    rescue Mess::TreeInvalidError
+      $stderr.puts "not an archive: #{target}"
+      exit 1
+    end
+  end
+
+  # plots
+  if $listing.include? :plots
+    puts
+    list_plots Mess::Plot
+  end
+
+  # none
+  if $listing.none?
+    $stderr.puts 'no specification flag for listing'
+    exit 1
+  end
+
+  puts
+  exit
+
+end
+
+begin
+
+  # given directly chat
+  chat = Mess::Chat.new(target)
+
+rescue Mess::ChatInvalidError
+
+  # given tree
+  begin
+    tree = Mess::Tree.new(target, archived: opts.archived?)
+  rescue Mess::TreeInvalidError
+    $stderr.puts "not a chat/archive: #{target}"
+    exit 1
+  end
+
+  # select chat
+  input  = opts[:chat].join
+  chat   = nil
+  failed = false
+  loop do
+
+    # search for nick or index
+    chat   = tree.chats.find { |c| c.title == input }
+    chat ||= tree.chats[input.to_i - 1] if input.to_i.to_s == input and (1..tree.chats.size).include? input.to_i
+    unless chat.nil?
+      puts if failed
+      break
+    end
+
+    # failed
+    puts
+    list_chats tree unless failed
+    failed = true
+
+    print 'select> '
+    input = $stdin.gets.chomp
+
+  end
+
+end
+
+# select plot
+input  = opts[:plot]
+plot   = nil
+failed = false
+loop do
+
+  # map indexes to names
+  input.map! do |p|
+    if p.to_i.to_s == p and (1..Mess::Plot.available.size).include? p.to_i
+      Mess::Plot.available.keys[p.to_i - 1]
+    else
+      p
+    end
+  end
+
+  # validate names
+  if input.any? and input.all? { |p| Mess::Plot.available.include? p }
+    plot = Set.new(input)
+    puts if failed
+    break
+  end
+
+  # failed
+  puts
+  list_plots Mess::Plot unless failed
+  failed = true
+  print 'select> '
+  input = $stdin.gets.strip.split(/[, ]/).compact
+
+end
+
+# prepare
+puts 'preparing...'
+i = 1
+plot.map! do |p|
+  $stdout.cursor_up 1
+  puts "preparing... #{i}/#{plot.size}"
+  Mess.const_get(p).new(chat)
+end
+$stdout.cursor_up 1
+puts 'preparing... done'
+
+# analyze
+total = chat.count
+pcapx = total / 100
+check = 0
+chat.msgs.each_with_index do |m, i|
+  if i >= check
+    $stdout.cursor_up 1
+    puts "analyzing... #{100 * i / total}%  "
+    check += pcapx
+  end
+  plot.each do |p|
+    p.push m #rescue nil
+  end
+end
+$stdout.cursor_up 1
+puts 'analyzing... done'
+
+# review
+puts
+plot.each_with_index do |p, i|
+  $stdout.cursor_up 1
+  puts "reviewing... #{i + 1}/#{plot.size}"
+  p.review
+end
+$stdout.cursor_up 1
+puts 'reviewing... done'
+
+# export
+$stdout.cursor_up 1
+puts 'exporting...'
+data = Hash.new
+plot.each do |p|
+  data[p.class.name] = p.data
+end
+$stdout.cursor_up 1
+
+File.open(opts[:output], 'w') do |f|
+  f.puts data.to_json
+end
+
+puts 'exporting... done'

data/lib/mess.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'json'
+require 'date'
+require_relative 'mess/version'
+
+module Mess
+  class Error < StandardError; end
+  # Your code goes here...
+end
+
+require_relative 'mess/error'
+require_relative 'mess/plot'
+Dir["#{__dir__}/mess/**/*.rb"].each { |f| require_relative f }

data/lib/mess/chat.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Mess
+  class Chat
+
+    attr_reader :path, :title, :usrs
+
+    def initialize path
+      @path = File.expand_path path
+
+      # make sure path exists
+      unless File.exists? @path
+        raise ChatInvalidError
+      end
+
+      # if file, scope out to the whole chat dir
+      if File.file? @path
+        @path = File.dirname @path
+      end
+
+      @title = nil
+      @usrs  = nil
+      @msgs  = Array.new
+
+      # exactly one message_1.ext file should exist
+      tmp = Dir["#@path/message_1\\.*"]
+      raise ChatInvalidError unless tmp.one? 
+      tmp = tmp.first
+
+      # also store its extension
+      ext = File.extname tmp # ".json" or ".html"...
+      ext = ext[1..-1]       #  "json" or  "html"...
+
+      # reference the get_ext function
+      get = method("get_#{ext}")
+
+      # chronologically iterate through message_X.ext chat files
+      # decrementally for some reason reee
+      total = Dir["#@path/message_*\\.#{ext}"].size
+      for i in (0...total)
+        file = "#@path/message_#{total - i}.#{ext}"
+        get.call(file) rescue raise ChatInvalidError
+      end
+
+      @title = '*you*' if @title.empty?
+      @ready = false
+
+    end
+
+    def count
+      @msgs.size
+    end
+
+    def prepare
+      @msgs.sort! { |a, b| a['timestamp_ms'] <=> b['timestamp_ms'] }
+    end
+
+    def msgs
+      prepare unless @ready
+      @ready = true
+      @msgs
+    end
+
+    private
+
+    def get_json file
+      buffer   = JSON.parse(File.read(file))
+      @title ||= buffer['title']
+      @usrs  ||= buffer['participants'].map{ |p| p['name'] }.sort
+      @msgs   += buffer['messages']
+    end
+
+    def get_html file
+    end
+
+  end
+end

data/lib/mess/error.rb

@@ -0,0 +1,12 @@
+module Mess
+
+  class Error < StandardError
+  end
+
+  class ChatInvalidError < Error
+  end
+
+  class TreeInvalidError < Error
+  end
+
+end

data/lib/mess/plot.rb

@@ -0,0 +1,30 @@
+module Mess
+  class Plot
+
+    @@available = Hash.new
+
+    def self.available
+      @@available
+    end
+
+    def self.describe desc
+      @@available[name] = desc
+    end
+
+    def self.name
+      self.to_s.match(/Mess::(?<name>\w+)$/)[:name]
+    end
+
+    attr_reader :data
+
+    def initialize chat
+    end
+
+    def push msg
+    end
+
+    def review
+    end
+
+  end
+end

data/lib/mess/plot/daily_msgs.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Mess
+  class DailyMsgs < Plot
+
+    describe 'daily messages by user'
+
+    def initialize chat
+      @data  = Hash.new
+      @usrs  = chat.usrs
+      oldest = get_day(chat.msgs[0])
+      latest = get_day(chat.msgs[-1])
+      for day in (oldest..latest)
+        @data[day.to_s] = Array.new(@usrs.size, 0)
+      end
+    end
+
+    def push msg
+      usr = msg['sender_name']
+      day = get_day(msg).to_s
+      @data[day][@usrs.index(usr)] += 1
+    end
+
+    private
+
+    def get_day msg
+      unix = msg['timestamp_ms'] / 1000
+      Date.parse(Time.at(unix).to_s)
+    end
+
+  end
+end

data/lib/mess/plot/total_delay.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Mess
+  class TotalDelay < Plot
+
+    describe 'total delay by user'
+
+    def initialize chat
+      @last = nil
+      @time = Hash.new
+      @data = Hash.new
+      chat.usrs.each do |u|
+        @data[u] = 0
+      end
+    end
+
+    def push msg
+
+      usr  = msg["sender_name"]
+      time = msg["timestamp_ms"]
+
+      if @time[usr].to_i < time
+        @time[usr] = time
+      end
+
+      if @last.nil? or @last == usr
+        @last ||= usr
+        return
+      end
+
+      @data[usr] += time - @time[@last]
+      @last = usr
+
+    end
+
+  end
+end

data/lib/mess/plot/total_lettres.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mess
+  class TotalLetters < Plot
+
+    describe 'total letters per user'
+
+    def initialize chat
+      @data = Hash.new
+      chat.usrs.each do |u|
+        @data[u] = 0
+      end
+    end
+
+    def push msg
+      usr = msg['sender_name']
+      letters = msg['content'].to_s.length
+      @data[usr] += letters
+    end
+
+  end
+end

data/lib/mess/plot/total_msgs.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mess
+  class TotalMsgs < Plot
+
+    describe 'total messages by user'
+
+    def initialize chat
+      @data = Hash.new
+      chat.usrs.each do |u|
+        @data[u] = 0
+      end
+    end
+
+    def push msg
+      usr = msg['sender_name']
+      @data[usr] += 1
+    end
+
+  end
+end

data/lib/mess/tree.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Mess
+  class Tree
+
+    attr_reader :root, :chats
+
+    def initialize path, flags = Hash.new
+
+      # find the root from given path
+
+      path = File.expand_path(path)
+      begin 
+
+        unless File.exists? path
+          break
+        end
+
+        if check(path)
+          @root = path
+          break
+        else
+          next
+        end 
+
+      end until (path = File.dirname(path)) == "/"
+
+      raise TreeInvalidError if @root.nil?
+      
+      import flags[:archived]
+
+    end
+
+    def check root
+      [
+        '/',
+        '/inbox',
+        '/archived_threads',
+      ].all? { |dir| File.directory?(root + dir) }
+    end
+
+    private
+
+    def import archived
+
+      @chats = Array.new
+
+      box = Array.new
+      box << 'inbox'
+      box << 'archived_threads' if archived
+
+      # find all text chat dirs
+      box.each do |b|
+        Dir["#@root/#{b}/*"].each do |dir|
+
+          # exclude asset dirs (capitalized)
+          base = File.basename dir
+          next unless base.downcase == base
+
+          # list it
+          begin
+            @chats << Chat.new(dir)
+          rescue
+          end
+
+        end
+      end
+
+      @chats.sort! { |a, b| a.title <=> b.title }
+
+    end
+
+    def select i
+      Chat.new(@index[i.to_i].to_s)
+    end
+
+  end
+end

data/lib/mess/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mess
+  VERSION = '0.0.0'
+end

data/mess.gemspec

@@ -0,0 +1,25 @@
+require_relative 'lib/mess/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mess"
+  spec.version       = Mess::VERSION
+  spec.authors       = ["patztablook22"]
+  spec.email         = ["patz@tuta.io"]
+
+  spec.summary       = "Facebook Messenger Statistician"
+  spec.description   = "Analyze your archived Messenger chatlogs in JSON format!"
+  spec.homepage      = "https://github.com/patztablook22/mess"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(__dir__) do
+    Dir["**/*"].reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "slop", "~> 4.8"
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: mess
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- patztablook22
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-10-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.8'
+description: Analyze your archived Messenger chatlogs in JSON format!
+email:
+- patz@tuta.io
+executables:
+- mess
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/mess
+- lib/mess.rb
+- lib/mess/chat.rb
+- lib/mess/error.rb
+- lib/mess/plot.rb
+- lib/mess/plot/daily_msgs.rb
+- lib/mess/plot/total_delay.rb
+- lib/mess/plot/total_lettres.rb
+- lib/mess/plot/total_msgs.rb
+- lib/mess/tree.rb
+- lib/mess/version.rb
+- mess.gemspec
+- pkg/mess-0.0.0.gem
+homepage: https://github.com/patztablook22/mess
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Facebook Messenger Statistician
+test_files: []