csgolytics 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3660933561cc7673f3b1eda381fc034a06d575a0
+  data.tar.gz: 01d2a323968d64b29d731952dcdc71910e92044f
+SHA512:
+  metadata.gz: df2675b4924ea7fb7297885bfeb18174fe77675ceedacbca8b0d1a1d511a84aeac03d18b5521f44b82f7fc36dd59b6dd805baac646fad23e8f2159c1dfc038e2
+  data.tar.gz: 1e26a07e7b195c4170213b7afb4082a5828eac5e2ba9d0c76795d2f0cc937e611f34bd1ff53b15933605157f5ae13e08266d754aac83034d5d56f26ad14e825c

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2016 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use, copy and modify copies of the Software, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+CSGOLytics
+==========
+
+CSGOLytics imports the Valve SRCDS/CS:GO (Counter Strike Global Offense) server logfile and writes all game events as structured data (JSON) into a backend database ([EventQL](https://eventql.io/)).
+
+![Screenshot](https://raw.githubusercontent.com/paulasmuth/csgolytics/master/screenshot.png)
+
+Installation
+------------
+
+The most simple way to install CSGOLytics is by using rubygems:
+
+    $ gem install csgolytics
+    
+Usage
+-----
+
+CSGOLytics requires a running [EventQL](https://eventql.io/) database. To use CSGOLytics, you also have to enable detailed logging in your dedicated server. You can either execute these lines via rcon or put them into your `autoexec.cfg` file.
+
+    > rcon log on
+    > rcon mp_logdetail 3
+
+#### Sending log data via logtail
+
+The preferred way of importing the CS:GO logfiles is by using the included `csgolytics-import` tool. To start the import tool, execute this command:
+
+    $ ./csgolytics-import \
+          --logdir /path/to/csgo/server/csgo/logs \
+          --eventql_host localhost \
+          --eventql_port 9175 \
+          --eventql_database csgolytics
+    
+#### Sending log data via UDP
+
+Alternatively, you can use the built-in remote logging support to receive the log data via udp. However this is less reliable. To use UDP logging, start the import tool with this command.
+    
+    $ ./csgolytics-import \
+          --listen_udp 0.0.0.0:3764 \
+          --eventql_host localhost \
+          --eventql_port 9175 \
+          --eventql_database csgolytics
+         
+
+Then execute this line via rcon or put it into the `autoexec.cfg` file (replace x.x.x.x with the address on which csgolytics-import is listening for udp packets)
+
+    > rcon logaddress_add x.x.x.x:3764
+    
+
+
+Game Events (JSON)
+------------------
+
+#### Frag Event (csgo_frags)
+
+    {  
+       "event":"frag",
+       "time":"2016-12-18T11:51:19Z",
+       "attacker_name":"le_fnord",
+       "attacker_steamid":"STEAM_1:0:123456789",
+       "attacker_team":"CT",
+       "attacker_coords_x":-285,
+       "attacker_coords_y":343,
+       "attacker_coords_z":1,
+       "victim_name":"Allen",
+       "victim_steamid":"BOT",
+       "victim_team":"T",
+       "victim_coords_x":-282,
+       "victim_coords_y":424,
+       "victim_coords_z":125,
+       "weapon":"ak47",
+       "headshot":false,
+       "penetrated":false,
+       "server_id":"aim"
+    }
+
+#### Assist Event (csgo_assists)
+
+    {  
+       "event":"assist",
+       "time":"2016-12-18T11:51:19Z",
+       "attacker_name":"le_fnord",
+       "attacker_steamid":"STEAM_1:0:123456789",
+       "attacker_team":"CT",
+       "victim_name":"Allen",
+       "victim_steamid":"BOT",
+       "victim_team":"T"
+    }
+
+
+License
+-------
+
+Copyright (c) 2016 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/csgolytics-import

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+$: << ::File::expand_path("../../lib", __FILE__)
+
+require "rubygems"
+require "yaml"
+require 'optparse'
+require 'eventql'
+require 'webrick'
+require "csgolytics/schema_manager"
+require "csgolytics/feed_upload"
+require "csgolytics/log_reader"
+
+# parse flags
+flags = {}
+
+flag_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: csgolytics [options]"
+
+  opts.on("--logdir=PATH", "/path/to/csgo/log") do |val|
+    flags[:logdir] = val
+  end
+
+  opts.on("--server_id=ID", "my_csgo_server_id") do |val|
+    flags[:server_id] = val
+  end
+
+  opts.on("--eventql_host=HOST", "EventQL host") do |val|
+    flags[:eventql_host] = val
+  end
+
+  opts.on("--eventql_port=PORT", "EventQL port") do |val|
+    flags[:eventql_port] = val
+  end
+
+  opts.on("--eventql_database=DATABASE", "EventQL database") do |val|
+    flags[:eventql_database] = val
+  end
+
+  opts.on("--eventql_auth_token=TOKEN", "EventQL auth_token") do |val|
+    flags[:eventql_auth_token] = val
+  end
+
+  opts.on("-v", "--[no-]verbose", "Print debug output") do |val|
+    flags[:verbose] = val
+  end
+
+  opts.on("-h", "--help", "You're reading it") do
+    puts opts
+    exit 0
+  end
+
+end
+
+flag_parser.parse!
+
+# check arguments
+unless File.directory?(flags[:logdir])
+  $stderr.puts "ERROR: #{flags[:logdir]} is not a directory"
+  exit 1
+end
+
+# init eventql client
+$stderr.puts "Connecting to EventQL at #{flags[:eventql_host]}"
+db = EventQL.connect(
+  :host => flags[:eventql_host],
+  :port => flags[:eventql_port],
+  :database => flags[:eventql_database],
+  :auth_token => flags[:eventql_auth_token]
+)
+
+# migrate database
+schema_manager = CSGOLytics::SchemaManager.new(
+    db,
+    ::File::expand_path("../../db", __FILE__))
+
+schema_manager.migrate!
+
+# start feed upload
+feed_upload = CSGOLytics::FeedUpload.new(db, flags[:server_id])
+
+# start log reader
+log_reader = CSGOLytics::LogReader.new(flags[:logdir])
+log_reader.on_logline do |l|
+  feed_upload.insert_logline(l)
+end
+
+log_reader.start
+
+

data/db/csgo_assists.sql

@@ -0,0 +1,13 @@
+CREATE TABLE csgo_assists (
+  time_key uint64,
+  event_id string,
+  server_id string,
+  time datetime,
+  attacker_name string,
+  attacker_steamid string,
+  attacker_team string,
+  victim_name string,
+  victim_steamid string,
+  victim_team string,
+  PRIMARY KEY (time_key, event_id)
+) WITH user_defined_partitions = "true";

data/db/csgo_frags.sql

@@ -0,0 +1,17 @@
+CREATE TABLE csgo_frags (
+  time_key uint64,
+  event_id string,
+  server_id string,
+  time datetime,
+  attacker_name string,
+  attacker_steamid string,
+  attacker_team string,
+  victim_name string,
+  victim_steamid string,
+  victim_team string,
+  weapon string,
+  headshoot bool,
+  penetrated bool,
+  distance double,
+  PRIMARY KEY (time_key, event_id)
+) WITH user_defined_partitions = "true";

data/lib/csgolytics/feed_upload.rb

@@ -0,0 +1,53 @@
+require "securerandom"
+require "csgolytics/log_parser"
+
+module CSGOLytics; end
+
+class CSGOLytics::FeedUpload
+
+  PARTITION_SIZE = 3600 * 24
+
+  EVENT_TABLE_MAP = {
+    "frag" => "csgo_frags",
+    "assist" => "csgo_assists"
+  }
+
+  def initialize(db, server_id = nil)
+    @db = db
+    @server_id = server_id
+    @log_parser = CSGOLytics::LogParser.new
+  end
+
+  def insert_logline(logline, event_id = nil)
+    ev = @log_parser.parse(logline)
+    unless ev
+      return
+    end
+
+    if event_id
+      ev[:event_id] = event_id
+    else
+      ev[:event_id] = Digest::SHA1.hexdigest logline
+    end
+
+    if @server_id
+      ev[:server_id] = @server_id
+    end
+
+    upload_event(ev)
+  end
+
+private
+
+  def upload_event(ev)
+    table = EVENT_TABLE_MAP[ev[:event]]
+    unless table
+      return
+    end
+
+    ev[:time_key] = (Time.parse(ev[:time]).to_i / PARTITION_SIZE) * PARTITION_SIZE
+
+    @db.insert!([{ :table => table, :database => @db.get_database, :data => ev }])
+  end
+
+end

data/lib/csgolytics/http_server.rb

@@ -0,0 +1,41 @@
+module CSGOLytics; end
+
+class CSGOLytics::HTTPServlet < WEBrick::HTTPServlet::AbstractServlet
+
+  def initialize(server, backend)
+    super server
+    @backend = backend
+  end
+
+  def do_POST (request, response)
+    if request.path == "/api/v1/insert_logline"
+      @backend.insert_logline request.body.force_encoding("utf-8")
+      response.status = 201
+      response.body = "ok"
+      response.content_type = "text/plan"
+      return
+     end
+
+    response.status = 404
+    response.body = "not found"
+    response.content_type = "text/plan"
+  end
+
+end
+
+class CSGOLytics::HTTPServer
+
+  def initialize(backend, port)
+    @http = WEBrick::HTTPServer.new(:Port => port)
+    @http.mount "/", CSGOLytics::HTTPServlet, backend
+  end
+
+  def listen
+    @http.start
+  end
+
+  def shutdown
+    @http.shutdown
+  end
+
+end

data/lib/csgolytics/log_listener.rb

@@ -0,0 +1,62 @@
+require "socket"
+
+module CSGOLytics; end
+
+class CSGOLytics::LogListener
+
+  REMOTE_LOG_PKT_HEADER = "\xff\xff\xff\xffR"
+
+  def initialize(config)
+    @ssock = UDPSocket.new
+    @ssock.bind("0.0.0.0", 3764)
+    @callbacks = []
+
+    @remotes_map = {}
+    config["servers"].each do |server|
+      server["remote_addrs"].each do |remote_addr|
+        @remotes_map[remote_addr] = server["server_id"]
+      end
+    end
+  end
+
+  def listen
+    while msg = @ssock.recvfrom(0xffff)
+      payload, raddr = msg
+      payload = payload.force_encoding("utf-8")
+
+      remote_addrs = [
+        "#{raddr[2]}:#{raddr[1]}",
+        "#{raddr[3]}:#{raddr[1]}"
+      ].uniq
+
+      remote_server_id = nil
+      remote_addrs.each do |r|
+      remote_server_id ||= @remotes_map[r]
+      end
+
+      unless remote_server_id
+        $stderr.puts "WARNING: unknown server: #{remote_addrs.inspect}"
+        next
+      end
+
+      unless payload.start_with?(REMOTE_LOG_PKT_HEADER)
+        $stderr.puts "WARNING: received invalid log packet"
+        next
+      end
+
+      logline = payload[REMOTE_LOG_PKT_HEADER.length..-1]
+      while ["\0", "\r", "\n"].include?(logline[-1])
+        logline = logline[0..-2]
+      end
+
+      @callbacks.each do |cb|
+        cb[logline, remote_server_id]
+      end
+    end
+  end
+
+  def on_logline(&cb)
+    @callbacks << cb
+  end
+
+end

data/lib/csgolytics/log_parser.rb

@@ -0,0 +1,79 @@
+require "socket"
+require "date"
+require "time"
+
+module CSGOLytics; end
+
+class CSGOLytics::LogParser
+
+  def parse(line)
+    ev = nil
+    return ev if ev = parse_frag(line)
+    return ev if ev = parse_assist(line)
+    return ev
+  end
+
+private
+
+  def parse_frag(line)
+    r = /^L (?<time>\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}:\d{2}): "(?<attacker_name>.*)<\d+><(?<attacker_steamid>BOT|(STEAM_\d+:\d+:\d+))><(?<attacker_team>CT|TERRORIST)>" \[(?<attacker_coords>-?\d+ -?\d+ -?\d+)\] killed "(?<victim_name>.*)<\d+><(?<victim_steamid>BOT|(STEAM_\d+:\d+:\d+))><(?<victim_team>CT|TERRORIST)>" \[(?<victim_coords>-?\d+ -?\d+ -?\d+)\] with "(?<weapon>\w+)" ?\(?(?<headshot>headshot)? ?(?<penetrated>penetrated)?\)?$/
+
+    m = r.match(line)
+    if m
+      attacker_coords = m[:attacker_coords].split
+      victim_coords = m[:victim_coords].split
+
+      return {
+        :event => "frag",
+        :time => parse_time(m[:time]).utc.iso8601,
+        :attacker_name => m[:attacker_name],
+        :attacker_steamid => m[:attacker_steamid],
+        :attacker_team => normalize_team(m[:attacker_team]),
+        :attacker_coords_x => attacker_coords[0].to_i,
+        :attacker_coords_y => attacker_coords[1].to_i,
+        :attacker_coords_z => attacker_coords[2].to_i,
+        :victim_name => m[:victim_name],
+        :victim_steamid => m[:victim_steamid],
+        :victim_team => normalize_team(m[:victim_team]),
+        :victim_coords_x => victim_coords[0].to_i,
+        :victim_coords_y => victim_coords[1].to_i,
+        :victim_coords_z => victim_coords[2].to_i,
+        :weapon => m[:weapon],
+        :headshot => !!m[:headshot],
+        :penetrated => !!m[:penetrated]
+      }
+    end
+  end
+
+  def parse_assist(line)
+    r = /^L (?<time>\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}:\d{2}): "(?<attacker_name>.*)<\d+><(?<attacker_steamid>BOT|(STEAM_\d+:\d+:\d+))><(?<attacker_team>CT|TERRORIST)>" assisted killing "(?<victim_name>.*)<\d+><(?<victim_steamid>BOT|(STEAM_\d+:\d+:\d+))><(?<victim_team>CT|TERRORIST)>"$/
+
+    m = r.match(line)
+    if m
+      return {
+        :event => "assist",
+        :time => parse_time(m[:time]).utc.iso8601,
+        :attacker_name => m[:attacker_name],
+        :attacker_steamid => m[:attacker_steamid],
+        :attacker_team => normalize_team(m[:attacker_team]),
+        :victim_name => m[:victim_name],
+        :victim_steamid => m[:victim_steamid],
+        :victim_team => normalize_team(m[:victim_team])
+      }
+    end
+  end
+
+  def parse_time(time_str)
+    DateTime.strptime("#{time_str} #{Time.now.strftime("%:z")}", "%m/%d/%Y - %H:%M:%S %z").to_time
+  end
+
+  def normalize_team(team)
+    case team
+      when "CT" then return "CT"
+      when "T" then return "T"
+      when "TERRORIST" then return "T"
+      else return "?"
+    end
+  end
+
+end

data/lib/csgolytics/log_reader.rb

@@ -0,0 +1,75 @@
+require "fileutils"
+require 'optparse'
+require 'uri'
+require 'net/http'
+require 'digest/sha1'
+
+module CSGOLytics; end
+
+class CSGOLytics::LogReader
+
+  POLL_INTERVAL = 1
+
+  def initialize(logdir)
+    @logdir = logdir
+    @callbacks = []
+  end
+
+  def on_logline(&cb)
+    @callbacks << cb
+  end
+
+  def start
+    # tail the logfiles
+    loop do
+      begin
+        # list all logfiles
+        @logfiles = {}
+        @logfiles_completed = {}
+        Dir.entries(@logdir).each do |f|
+          if f.end_with?(".log")
+            @logfiles[f] = true
+          end
+
+          if f =~ /^\.(.*\.log)\.csgolytics_complete$/
+            @logfiles_completed[$1] = true
+          end
+        end
+
+        @logfiles = (@logfiles.keys - @logfiles_completed.keys).sort
+
+        # read each logfile from the last offset and upload new lines
+        @logfiles.each_with_index do |fname, logfile_index|
+          fpath = File.join(@logdir, fname)
+          offset_file_path = File.join(@logdir, ".#{fname}.csgolytics_offset")
+          offset = 0
+          if File.exists?(offset_file_path)
+            offset = IO.read(offset_file_path).to_i
+          end
+
+          puts "Reading #{fpath} from #{offset}"
+
+          f = File.open(fpath)
+          f.seek(offset)
+          while l = (f.readline rescue nil)
+            @callbacks.each do |cb|
+              cb[l.chomp]
+            end
+
+            File.write(offset_file_path + "~", f.tell)
+            FileUtils.mv(offset_file_path + "~", offset_file_path)
+          end
+
+          if logfile_index < @logfiles.length - 1
+            FileUtils.touch(File.join(@logdir, ".#{fname}.csgolytics_complete"))
+          end
+        end
+      rescue Exception => e
+        $stderr.puts "ERROR: #{e.to_s}"
+      end
+
+      sleep POLL_INTERVAL
+    end
+  end
+
+end

data/lib/csgolytics/schema_manager.rb

@@ -0,0 +1,44 @@
+require "socket"
+
+module CSGOLytics; end
+
+class CSGOLytics::SchemaManager
+
+  def initialize(db, schema_path)
+    @db = db
+    @schema_path = schema_path
+
+    @tables = []
+    Dir.entries(schema_path).each do |tbl|
+      next if tbl.start_with?(".")
+
+      if tbl =~ /(\w+)\.sql/
+        @tables << $1
+      else
+        $stderr.puts "ERROR: invalid table schema file #{tbl}"
+        exit 1
+      end
+    end
+  end
+
+  def migrate!
+    $stderr.puts "Migrating database schemas, this may take a second"
+    list_tables_qry = @db.query("show tables;")
+    list_tables_res = list_tables_qry.execute!
+
+    actual_tables = list_tables_res[0]["rows"].map(&:first)
+    (@tables - actual_tables).each do |tbl|
+      create_table!(tbl)
+    end
+  end
+
+private
+
+  def create_table!(tbl)
+    $stderr.puts "Creating table: #{tbl}"
+    create_table_sql = IO.read(File::join(@schema_path, tbl + ".sql"))
+    create_table_qry = @db.query(create_table_sql)
+    create_table_qry.execute!
+  end
+
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: csgolytics
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Paul Asmuth
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-19 00:00:00.000000000 Z
+dependencies: []
+description: CS:GO Analaytics with EventQL
+email: paul@eventql.io
+executables:
+- csgolytics-import
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/csgolytics-import
+- db/csgo_assists.sql
+- db/csgo_frags.sql
+- lib/csgolytics/feed_upload.rb
+- lib/csgolytics/http_server.rb
+- lib/csgolytics/log_listener.rb
+- lib/csgolytics/log_parser.rb
+- lib/csgolytics/log_reader.rb
+- lib/csgolytics/schema_manager.rb
+homepage: http://github.com/eventql/eventql
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: CS:GO Analaytics with EventQL
+test_files: []