wafris 0.0.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd571cbf8934f0b14384e1b85d42d5a70ffaaa98f17e8fcfd3545553b6b1d6af
-  data.tar.gz: a5a053621c83c0ed2892a8aca08fc26439e6e0cff95bcdbf8787b35c0ad1eaf3
+  metadata.gz: 05707fe625f71f24a02f26585d89356156b1dc83887195f1042a47856d3c86cf
+  data.tar.gz: 05e798e3298518a0310f1cdf6a68426b2b12f9c066acc74e78833987e26f2e31
 SHA512:
-  metadata.gz: 339e8b1fee46d79287716e20edfc24b80c720a82f12d07485508c0beb16c9813f4f35530b7473d0711d75a6e316e97236d7912024b984e96524649021d58be19
-  data.tar.gz: c674ca5de599c5f0bb7f7bfb6beba19d4bec38411f66cddf27897c4ecafe9a3486e723b30e39a7cb63472106ca62483ee7ffc5b98fd31dd331d7eae82efc13df
+  metadata.gz: b41d4ee2190f5a88e263ef6905ffe43e6727923df4f3f1a600f203d10ea2010409e95c02c965cc2161fce47d5d541aaa1217beeb05097134fcaecdcfcd4305f1
+  data.tar.gz: dd940d846b7892a4607c78767f2ca364c62ab33782deb76b863d8b393f1a5c82134f0b3f6c76a438f9993e7d7702df67899d9e6485bc5491e03fcc3e02a38c1e

data/lib/lua/dist/get_graph_data.lua

@@ -0,0 +1,81 @@
+-- Compiled from:
+-- src/get_time_buckets.lua
+
+-- Code was pulled from https://otland.net/threads/how-convert-timestamp-to-date-type.251657/
+-- An alternate solution is https://gist.github.com/markuman/e96d04139cd8acc33604
+local function get_time_bucket_from_timestamp(unix_time_milliseconds)
+  local function calculate_years_number_of_days(yr)
+    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+  end
+
+  local function get_year_and_day_number(year, days)
+    while days >= calculate_years_number_of_days(year) do
+      days = days - calculate_years_number_of_days(year)
+      year = year + 1
+    end
+    return year, days
+  end
+
+  local function get_month_and_month_day(days, year)
+    local days_in_each_month = {
+      31,
+      (calculate_years_number_of_days(year) == 366 and 29 or 28),
+      31,
+      30,
+      31,
+      30,
+      31,
+      31,
+      30,
+      31,
+      30,
+      31,
+    }
+
+    for month = 1, #days_in_each_month do
+      if days - days_in_each_month[month] <= 0 then
+        return month, days
+      end
+      days = days - days_in_each_month[month]
+    end
+  end
+
+  local unix_time = unix_time_milliseconds / 1000
+  local year = 1970
+  local days = math.ceil(unix_time / 86400)
+  local month = nil
+
+  year, days = get_year_and_day_number(year, days)
+  month, days = get_month_and_month_day(days, year)
+  local hours = math.floor(unix_time / 3600 % 24)
+  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
+  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
+  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
+end
+
+local function get_time_buckets(unix_time_milliseconds)
+  local time_buckets = {}
+
+  for i = 23, 0, -1 do
+    table.insert(time_buckets, get_time_bucket_from_timestamp(unix_time_milliseconds - (1000 * 60 * 60 * i)))
+  end
+  return time_buckets
+end
+
+local function num_requests(time_bucket)
+  local request_keys = redis.call("KEYS", "unique-requests:" .. time_bucket)
+  redis.call("PFMERGE", "merged_unique-requests", unpack(request_keys))
+  return redis.call("PFCOUNT", "merged_unique-requests")
+end
+
+local graph_data = {}
+local unix_time_milliseconds = ARGV[1]
+local time_buckets = get_time_buckets(unix_time_milliseconds)
+-- use the get_time_buckets method to get each time bucket and
+-- the associated count for that bucket
+for bucket in pairs(time_buckets) do
+  table.insert(graph_data, time_buckets[bucket])
+  table.insert(graph_data, num_requests(time_buckets[bucket]))
+end
+
+return graph_data

data/lib/lua/dist/wafris_core.lua

@@ -0,0 +1,96 @@
+local function get_time_bucket_from_timestamp(unix_time_milliseconds)
+  local function calculate_years_number_of_days(yr)
+    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+  end
+
+  local function get_year_and_day_number(year, days)
+    while days >= calculate_years_number_of_days(year) do
+      days = days - calculate_years_number_of_days(year)
+      year = year + 1
+    end
+    return year, days
+  end
+
+  local function get_month_and_month_day(days, year)
+    local days_in_each_month = {
+      31,
+      (calculate_years_number_of_days(year) == 366 and 29 or 28),
+      31,
+      30,
+      31,
+      30,
+      31,
+      31,
+      30,
+      31,
+      30,
+      31,
+    }
+
+    for month = 1, #days_in_each_month do
+      if days - days_in_each_month[month] <= 0 then
+        return month, days
+      end
+      days = days - days_in_each_month[month]
+    end
+  end
+
+  local unix_time = unix_time_milliseconds / 1000
+  local year = 1970
+  local days = math.ceil(unix_time / 86400)
+  local month = nil
+
+  year, days = get_year_and_day_number(year, days)
+  month, days = get_month_and_month_day(days, year)
+  local hours = math.floor(unix_time / 3600 % 24)
+  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
+  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
+  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
+end
+
+-- For: Relationship of IP to time of Request (Stream)
+local function get_request_id(timestamp, ip, max_requests)
+  timestamp = timestamp or "*"
+  local request_id = redis.call("XADD", "ip-requests-stream", "MAXLEN", "~", max_requests, timestamp, "ip", ip)
+  return request_id
+end
+
+local function add_to_HLL_request_count(timebucket, request_id)
+  redis.call("PFADD", "unique-requests:" .. timebucket, request_id)
+end
+
+-- For: Leaderboard of IPs with Request count as score
+local function increment_timebucket_for_ip(timebucket, ip)
+  redis.call("ZINCRBY", "ip-leader-sset:" .. timebucket, 1, ip)
+end
+
+-- Configuration
+local max_requests = 100000
+local max_requests_per_ip = 10000
+
+local ip = ARGV[1]
+local ip_to_decimal = ARGV[2]
+local unix_time_milliseconds = ARGV[3]
+local unix_time = ARGV[3] / 1000
+
+-- Initialize local variables
+local request_id = get_request_id(nil, ip, max_requests)
+local current_timebucket = get_time_bucket_from_timestamp(unix_time_milliseconds)
+
+-- GRAPH DATA COLLECTION
+add_to_HLL_request_count(current_timebucket, request_id)
+
+-- LEADERBOARD DATA COLLECTION
+increment_timebucket_for_ip(current_timebucket, ip)
+
+-- BLOCKING LOGIC
+-- Safelist Range Check
+if next(redis.call("ZRANGEBYSCORE", "allowed_ranges", ip_to_decimal, "+inf", "LIMIT", 0, 1)) then
+  return "Allowed"
+-- Blocklist Range Check
+elseif next(redis.call("ZRANGEBYSCORE", "blocked_ranges", ip_to_decimal, "+inf", "LIMIT", 0, 1)) then
+  return "Blocked"
+-- No Matches
+else
+  return "Not found"
+end

data/lib/lua/src/get_time_buckets.lua

@@ -0,0 +1,58 @@
+function get_time_bucket_from_timestamp(unix_time_milliseconds)
+  local function calculate_years_number_of_days(yr)
+    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+  end
+
+  local function get_year_and_day_number(year, days)
+    while days >= calculate_years_number_of_days(year) do
+      days = days - calculate_years_number_of_days(year)
+      year = year + 1
+    end
+    return year, days
+  end
+
+  local function get_month_and_month_day(days, year)
+    local days_in_each_month = {
+      31,
+      (calculate_years_number_of_days(year) == 366 and 29 or 28),
+      31,
+      30,
+      31,
+      30,
+      31,
+      31,
+      30,
+      31,
+      30,
+      31,
+    }
+
+    for month = 1, #days_in_each_month do
+      if days - days_in_each_month[month] <= 0 then
+        return month, days
+      end
+      days = days - days_in_each_month[month]
+    end
+  end
+
+  local unix_time = unix_time_milliseconds / 1000
+  local year = 1970
+  local days = math.ceil(unix_time / 86400)
+  local month = nil
+
+  year, days = get_year_and_day_number(year, days)
+  month, days = get_month_and_month_day(days, year)
+  local hours = math.floor(unix_time / 3600 % 24)
+  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
+  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
+  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
+end
+
+function get_time_buckets(unix_time_milliseconds)
+  local time_buckets = {}
+
+  for i = 23, 0, -1 do
+    table.insert(time_buckets, get_time_bucket_from_timestamp(unix_time_milliseconds - (1000 * 60 * 60 * i)))
+  end
+  return time_buckets
+end

data/lib/lua/src/queries.lua

@@ -0,0 +1,14 @@
+local function num_requests(start_time, end_time)
+  local request_keys = redis.call('KEYS', 'unique-requests:*')
+  redis.call('PFMERGE', 'merged_unique-requests', unpack(request_keys))
+  return redis.call('PFCOUNT', 'merged_unique-requests')
+end
+
+local function unique_ips(start_time, end_time)
+  local ip_keys = redis.call('KEYS', 'unique-ips:*')
+  redis.call('PFMERGE', 'merged_unique-ips', unpack(ip_keys))
+  return redis.call('PFCOUNT', 'merged_unique-ips')
+end
+
+redis.debug("Request count: ", num_requests(0, 10000000))
+redis.debug("IP request count: ", unique_ips(0, 10000000))

data/lib/lua/src/seeds/data_load.lua

@@ -0,0 +1,104 @@
+-- Template strings below are replaced with generated
+-- data from the ip_data_generator.rb script
+-- local ipArray = { }
+-- local timestampArray = { }
+-- redis.debug("Timestamp count: ", #timestampArray)
+
+local function get_time_bucket_from_timestamp(unix_time_milliseconds)
+  local function calculate_years_number_of_days(yr)
+    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+  end
+
+  local function get_year_and_day_number(year, days)
+    while days >= calculate_years_number_of_days(year) do
+      days = days - calculate_years_number_of_days(year)
+      year = year + 1
+    end
+    return year, days
+  end
+
+  local function get_month_and_month_day(days, year)
+    local days_in_each_month = {
+      31,
+      (calculate_years_number_of_days(year) == 366 and 29 or 28),
+      31,
+      30,
+      31,
+      30,
+      31,
+      31,
+      30,
+      31,
+      30,
+      31,
+    }
+
+    for month = 1, #days_in_each_month do
+      if days - days_in_each_month[month] <= 0 then
+        return month, days
+      end
+      days = days - days_in_each_month[month]
+    end
+  end
+
+  local unix_time = unix_time_milliseconds / 1000
+  local year = 1970
+  local days = math.ceil(unix_time / 86400)
+  local month = nil
+
+  year, days = get_year_and_day_number(year, days)
+  month, days = get_month_and_month_day(days, year)
+  local hours = math.floor(unix_time / 3600 % 24)
+  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
+  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
+  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
+end
+
+-- For: Relationship of IP to time of Request (Stream)
+local function get_request_id(timestamp, ip, max_requests)
+  timestamp = timestamp or "*"
+  local request_id = redis.call("XADD", "ip-requests-stream", "MAXLEN", "~", max_requests, timestamp, "ip", ip)
+  return request_id
+end
+
+local function add_to_HLL_request_count(timebucket, request_id)
+  redis.call("PFADD", "unique-requests:" .. timebucket, request_id)
+end
+
+-- Configuration
+local max_requests = 100000
+local max_requests_per_ip = 10000
+
+-- Interior of this for loop is what should go into wafris_core.lua
+for i = 1, #timestampArray do
+  -- Setup
+  local ip = ipArray[math.random(#ipArray)]
+  local timestamp = timestampArray[i]
+
+  local request_id = get_request_id(timestamp, ip, max_requests)
+
+  -- GRAPH DATA COLLECTION
+  local current_timebucket = get_time_bucket_from_timestamp(timestamp)
+  add_to_HLL_request_count(current_timebucket, request_id)
+
+  -- For: Looking up Requests an IP has made (Stream) / time of request
+  local ip_stream_key = "ip-stream:" .. ip
+  local ip_stream_id =
+    redis.call("XADD", ip_stream_key, "MAXLEN", "~", max_requests_per_ip, "*", "request_id", request_id)
+
+  -- For: Precalc of Number of Requests (Key)
+  local requests_count_key = "requests-count:" .. current_timebucket
+  redis.call("INCR", requests_count_key)
+
+  -- For: Precalc of Number of Requests from an IP (Key)
+  local ips_count_bucket_key = "ips-count:" .. ip .. ":" .. current_timebucket
+  redis.call("INCR", ips_count_bucket_key)
+
+  -- For: Precalc of Number of Unique IPs making Requests (HLL)
+  local ips_count_hll_key = "unique-ips:" .. current_timebucket
+  redis.call("PFADD", ips_count_hll_key, ip)
+
+  -- For: Leaderboard of IPs with Request count as score
+  local ip_leaderboard_sset_key = "ip-leader-sset:" .. current_timebucket
+  redis.call("ZINCRBY", ip_leaderboard_sset_key, 1, ip)
+end

data/lib/lua/src/time_bucket.lua

@@ -0,0 +1,40 @@
+-- Code was pulled from https://otland.net/threads/how-convert-timestamp-to-date-type.251657/
+-- An alternate solution is https://gist.github.com/markuman/e96d04139cd8acc33604
+local function get_time_bucket_from_timestamp(unix_time_milliseconds)
+  local function calculate_years_number_of_days(yr)
+    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+  end
+
+  local function get_year_and_day_number(year, days)
+    while days >= calculate_years_number_of_days(year) do
+      days = days - calculate_years_number_of_days(year)
+      year = year + 1
+    end
+    return year, days
+  end
+
+  local function get_month_and_month_day(days, year)
+    local days_in_each_month = {
+      31,
+      (calculate_years_number_of_days(year) == 366 and 29 or 28),
+      31, 30, 31,30,31,31,30,31,30,31
+    }
+
+    for month = 1, #days_in_each_month do
+      if days - days_in_each_month[month] <= 0 then return month, days end
+      days = days - days_in_each_month[month]
+    end
+  end
+
+  local unix_time = unix_time_milliseconds / 1000
+  local year = 1970
+  local days = math.ceil(unix_time/86400)
+  local month = nil
+
+  year, days = get_year_and_day_number(year, days)
+  month, days = get_month_and_month_day(days, year)
+  local hours = math.floor(unix_time / 3600 % 24)
+  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
+  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
+  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
+end

data/lib/wafris/configuration.rb

@@ -28,15 +28,29 @@ module Wafris
       CONNECTION_ERROR
     end
 
-    def script_sha
-      @script_sha ||= redis.script(:load, wafris_core)
+    def core_sha
+      @core_sha ||= redis.script(:load, wafris_core)
     end
 
     def wafris_core
+      read_lua_dist("wafris_core")
+    end
+
+    def graph_sha
+      @graph_sha ||= redis.script(:load, wafris_graph)
+    end
+
+    def wafris_graph
+      read_lua_dist("get_graph_data")
+    end
+
+    private
+
+    def read_lua_dist(filename)
       File.read(
         File.join(
           File.dirname(__FILE__),
-          'wafris_core.lua'
+          "../lua/dist/#{filename}.lua"
         )
       )
     end

data/lib/wafris/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wafris
-  VERSION = "0.0.1"
+  VERSION = "0.1.2"
 end

data/lib/wafris.rb

@@ -31,14 +31,13 @@ module Wafris
 
     def allow_request?(request)
       configuration.connection_pool.with do |conn|
-        time = Time.now
+        time = Time.now.to_f * 1000
         status = conn.evalsha(
-          configuration.script_sha,
+          configuration.core_sha,
           argv: [
             request.ip,
             IPAddr.new(request.ip).to_i,
-            time.to_i,
-            "all-ips:#{time.strftime('%Y-%m-%d')}:#{time.hour}"
+            time.to_i
           ]
         )
 
@@ -49,5 +48,61 @@ module Wafris
         end
       end
     end
+
+    def add_block(ip)
+      configuration.connection_pool.with do |conn|
+        conn.zadd(
+          'blocked_ranges',
+          IPAddr.new(ip).to_i,
+          ip
+        )
+      end
+    end
+
+    def remove_block(ip)
+      configuration.connection_pool.with do |conn|
+        conn.zrem(
+          'blocked_ranges',
+          ip
+        )
+      end
+    end
+
+    def request_buckets(_now)
+      graph_data = []
+      configuration.connection_pool.with do |conn|
+        time = Time.now.to_f * 1000
+        graph_data = conn.evalsha(
+          configuration.graph_sha,
+          argv: [
+            time.to_i
+          ]
+        )
+      end
+
+      return graph_data
+    end
+
+    def ips_with_num_requests
+      configuration.connection_pool.with do |conn|
+        return conn.zunion(
+          *leader_timebuckets,
+          0, -1, with_scores: true
+        )
+      end
+    end
+
+    private
+
+    def leader_timebuckets
+      timebuckets = []
+
+      time = Time.now.utc
+      24.times do |hours|
+        timebuckets << "ip-leader-sset:#{(time - 60 * 60 * hours).strftime("%Y-%m-%d-%H")}"
+      end
+
+      return timebuckets
+    end
   end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: wafris
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Micahel Buckbee
 - Ryan Castillo
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-06 00:00:00.000000000 Z
+date: 2023-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -179,23 +179,28 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 13.0.6
-description:
-email:
+description: 
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/lua/dist/get_graph_data.lua
+- lib/lua/dist/wafris_core.lua
+- lib/lua/src/get_time_buckets.lua
+- lib/lua/src/queries.lua
+- lib/lua/src/seeds/data_load.lua
+- lib/lua/src/time_bucket.lua
 - lib/wafris.rb
 - lib/wafris/configuration.rb
 - lib/wafris/middleware.rb
 - lib/wafris/railtie.rb
 - lib/wafris/version.rb
-- lib/wafris/wafris_core.lua
-homepage:
+homepage: 
 licenses:
-- MIT
+- Elastic-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -211,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.3.26
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Web application firewall for Rack apps
 test_files: []

data/lib/wafris/wafris_core.lua

@@ -1,42 +0,0 @@
-local LAST_REQUESTS_TIME = 'last_requests_time'
-local TWENTY_FOUR_HOURS = 86400
-
-local ip = ARGV[1]
-local ip_to_decimal = ARGV[2]
-local unix_time = ARGV[3]
-local expire_time = unix_time - TWENTY_FOUR_HOURS
-local ip_request_string = "ip-requests-" .. ip
-local hour_bucket = ARGV[4]
-
--- LEADERBOARD DATA COLLECTION
--- Add IP to last_requests_time key by integer timestamp
--- ZADD last_requets_time 1661356145 '192.168.1.1'
-redis.call('ZADD', LAST_REQUESTS_TIME, unix_time, ip)
--- Remove IP from last_requests_time if it has been there for 24 hours
--- ZREMRANGEBYSCORE last_requests_time 0 (1661356145 - 86400)
-redis.call('ZREMRANGEBYSCORE', LAST_REQUESTS_TIME, 0, expire_time)
--- Add IP to ip-requests-<ip> for leaderboard tracking
--- LPUSH ip-requests-192.168.1.1 1661356145
-redis.call('LPUSH', ip_request_string, unix_time)
--- Have the key expire in 24 hours
--- EXPIRE ip-requests-192.168.1.1 86400
-redis.call('EXPIRE', ip_request_string, TWENTY_FOUR_HOURS)
-
--- GRAPH DATA COLLECTION
--- Increment counter for hourly buckets
--- INC all-ips:2022-10-01:12
-redis.call('INCR', hour_bucket)
--- EXPIRE all-ips:2022-10-01:12 86400
-redis.call('EXPIRE', hour_bucket, TWENTY_FOUR_HOURS)
-
--- BLOCKING LOGIC
--- Safelist Range Check
-if next(redis.call('ZRANGEBYSCORE', 'allowed_ranges', ip_to_decimal, "+inf", "LIMIT", 0, 1)) then
-  return 'Allowed'
--- Blocklist Range Check
-elseif next(redis.call('ZRANGEBYSCORE', 'blocked_ranges', ip_to_decimal, "+inf", "LIMIT", 0, 1)) then
-  return 'Blocked'
--- No Matches
-else
-  return 'Not found'
-end