wafris 0.8.5 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66b7b15717346a4c6cef883d8615d539cfe9a7a9aca15b1f430b11faf510f378
-  data.tar.gz: be84cc1b2adea93af908914942d798ec56406dfbeff5caf063cf2d088b8e1299
+  metadata.gz: 8513adabe26d6d0e250902e0e623e2da68abb1eaab787dfc33a637636a8fdf61
+  data.tar.gz: e515c1d91decc74401976aa68b0a241c5d06368fe33d654e43cc67fe67337479
 SHA512:
-  metadata.gz: b43cc330f0729a0ce0a74a77670819a474b2f28b11fcd8c45ea4ac30531aff0158fae6e468810666403ec90966444f1fe3f604f501fd173cc2ede8d509f38c53
-  data.tar.gz: 6649a022d4213ca7ad57858aed9a33b9860a7a51b9387b4129d12fc349546ac1ab108ff969e3d9a07c70031104816017ec07f71b3854b2d5c216b74fe06944f2
+  metadata.gz: fa98c3073ef1e614be18e40c867c05e5e9fb715a0722d5c8ef7483f6c643d049da373877329b16f44c0faca571ea336c181941ba27017b9c2e58f6d53fc1571e
+  data.tar.gz: be8909dc34636dc416ceb6d27b068fd3e5e2e1756d5ecadf764910ad657459da7534eba72012b8d2ac3c91d51dd972d982f516758b1494a00b94582d6e4a12a3

data/lib/lua/dist/wafris_core.lua

@@ -1,136 +1,274 @@
-local version = "v0.8:"
-local wafris_prefix = "w:" .. version
+local use_timestamps_as_request_ids = true
+local EXPIRATION_IN_SECONDS = 86400
 
-local function get_time_bucket_from_timestamp(unix_time_milliseconds, minutes_flag)
-  local function calculate_years_number_of_days(yr)
-    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
+local function get_timebucket(timestamp_in_seconds)
+  local startOfHourTimestamp = math.floor(timestamp_in_seconds / 3600) * 3600
+  return tostring(startOfHourTimestamp)
+end
+
+local function set_property_value_id_lookups(property_abbreviation, property_value)
+  local value_key = property_abbreviation .. "V" .. property_value
+  local property_id = redis.call("GET", value_key)
+
+  if property_id == false then
+    property_id = redis.call("INCR", property_abbreviation .. "-id-counter")
+    redis.call("SET", value_key, property_id)
+    redis.call("SET", property_abbreviation .. "I" .. property_id, property_value)
+  else
+    redis.call("EXPIRE", value_key, EXPIRATION_IN_SECONDS)
+    redis.call("EXPIRE", property_abbreviation .. "I" .. property_id, EXPIRATION_IN_SECONDS)
   end
 
-  local function get_year_and_day_number(year, days)
-    while days >= calculate_years_number_of_days(year) do
-      days = days - calculate_years_number_of_days(year)
-      year = year + 1
+  return property_id
+end
+
+local function increment_leaderboard_for(property_abbreviation, property_id, timebucket)
+  local key = property_abbreviation .. "L" .. timebucket
+  redis.call("ZINCRBY", key, 1, property_id)
+  redis.call("EXPIRE", key, EXPIRATION_IN_SECONDS)
+end
+
+local function set_property_to_requests_list(property_abbreviation, property_id, request_id, timebucket)
+  local key = property_abbreviation .. "R" .. property_id .. "-" .. timebucket
+  redis.call("LPUSH", key, request_id)
+  redis.call("EXPIRE", key, EXPIRATION_IN_SECONDS)
+end
+
+local function ip_in_hash(hash_name, ip_address)
+  local found_ip = redis.call('HEXISTS', hash_name, ip_address)
+
+  if found_ip == 1 then
+    return ip_address
+  else
+    return false
+  end
+end
+
+local function ip_in_cidr_range(cidr_set, ip_decimal_lexical)
+  local higher_value = redis.call('ZRANGEBYLEX', cidr_set, '['..ip_decimal_lexical, '+', 'LIMIT', 0, 1)[1]
+  local lower_value = redis.call('ZREVRANGEBYLEX', cidr_set, '['..ip_decimal_lexical, '-', 'LIMIT', 0, 1)[1]
+
+  if not (higher_value and lower_value) then
+      return false
+  end
+
+  local higher_compare = higher_value:match('([^%-]+)$')
+  local lower_compare = lower_value:match('([^%-]+)$')
+
+  if higher_compare == lower_compare then
+      return lower_compare
+  else
+      return false
+  end
+end
+
+local function match_by_pattern(property_abbreviation, property_value)
+  local hash_name = "rules-blocked-" .. property_abbreviation
+  local patterns = redis.call('HKEYS', hash_name)
+
+  for _, pattern in ipairs(patterns) do
+    if string.find(property_value, pattern) then
+      return pattern
     end
-    return year, days
   end
 
-  local function get_month_and_month_day(days, year)
-    local days_in_each_month = {
-      31,
-      (calculate_years_number_of_days(year) == 366 and 29 or 28),
-      31,
-      30,
-      31,
-      30,
-      31,
-      31,
-      30,
-      31,
-      30,
-      31,
-    }
-
-    for month = 1, #days_in_each_month do
-      if days - days_in_each_month[month] <= 0 then
-        return month, days
+  return false
+end
+
+local function blocked_by_rate_limit(request_properties)
+  local rate_limiting_rules_values = redis.call('HKEYS', 'rules-blocked-rate-limits')
+
+  for i, rule_name in ipairs(rate_limiting_rules_values) do
+    local conditions_hash = redis.call('HGETALL', rule_name .. "-conditions")
+    local all_conditions_match = true
+
+    for j = 1, #conditions_hash, 2 do
+      local condition_key = conditions_hash[j]
+      local condition_value = conditions_hash[j + 1]
+
+      if request_properties[condition_key] ~= condition_value then
+        all_conditions_match = false
+        break
       end
-      days = days - days_in_each_month[month]
     end
-  end
 
-  local unix_time = unix_time_milliseconds / 1000
-  local year = 1970
-  local days = math.ceil(unix_time / 86400)
-  local month = nil
-
-  year, days = get_year_and_day_number(year, days)
-  month, days = get_month_and_month_day(days, year)
-  local hours = math.floor(unix_time / 3600 % 24)
-  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
-  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
-  if minutes_flag == false then
-    return string.format("%04d%02d%02d%02d", year, month, days, hours)
-  elseif minutes_flag == true then
-    local minutes = math.floor(unix_time / 60 % 60)
-    return string.format("%04d%02d%02d%02d%02d", year, month, days, hours, minutes)
+    if all_conditions_match then
+      local rule_settings_key = rule_name .. "-settings"
+      local limit, time_period, limited_by, rule_id = unpack(redis.call('HMGET', rule_settings_key, 'limit', 'time-period', 'limited-by', 'rule-id'))
+      local throttle_key = rule_name .. ":" .. limit .. "V" .. request_properties.ip
+      local new_value = redis.call('INCR', throttle_key)
+
+      if new_value == 1 then
+        redis.call('EXPIRE', throttle_key, tonumber(time_period))
+      end
+
+      if tonumber(new_value) >= tonumber(limit) then
+        return rule_id
+      else
+        return false
+      end
+    end
   end
 end
 
--- For: Relationship of IP to time of Request (Stream)
-local function get_request_id(timestamp, ip, max_requests)
-  timestamp = timestamp or "*"
-  local request_id = redis.call("XADD", "ip-requests-stream", "MAXLEN", "~", max_requests, timestamp, "ip", ip)
-  return request_id
+local function check_rules(functions_to_check)
+  for _, check in ipairs(functions_to_check) do
+    local rule = check.func(unpack(check.args))
+    local category = check.category
+
+    if type(rule) == "string" then
+      return rule, category
+    end
+  end
+
+  return false, false
 end
 
-local function add_to_graph_timebucket(timebucket, request_id)
-  local key = wafris_prefix .. "gr-ct:" .. timebucket
-  redis.call("PFADD", key, request_id)
-  -- Expire the key after 25 hours if it has no expiry
-  redis.call("EXPIRE", key, 90000)
+local function check_blocks(request)
+  local rule_categories = {
+    { category = "bi", func = ip_in_hash, args = { "rules-blocked-i", request.ip } },
+    { category = "bc", func = ip_in_cidr_range, args = { "rules-blocked-cidrs-set", request.ip_decimal_lexical } },
+    { category = "bu", func = match_by_pattern, args = { "u", request.user_agent } },
+    { category = "bp", func = match_by_pattern, args = { "p", request.path } },
+    { category = "ba", func = match_by_pattern, args = { "a", request.parameters } },
+    { category = "bh", func = match_by_pattern, args = { "h", request.host } },
+    { category = "bm", func = match_by_pattern, args = { "m", request.method } },
+    { category = "bd", func = match_by_pattern, args = { "rh", request.headers } },
+    { category = "bpb", func = match_by_pattern, args = { "pb", request.post_body } },
+    { category = "brl", func = blocked_by_rate_limit, args = { request } }
+  }
+
+  return check_rules(rule_categories)
 end
 
--- For: Leaderboard of IPs with Request count as score
-local function increment_timebucket_for(type, timebucket, property)
-  local key = wafris_prefix .. type .. "lb:" .. timebucket
-  redis.call("ZINCRBY", key, 1, property)
-  -- Expire the key after 25 hours if it has no expiry
-  redis.call("EXPIRE", key, 90000)
+local function check_allowed(request)
+  local rule_categories = {
+    { category = "ai", func = ip_in_hash, args = { "rules-allowed-i", request.ip } },
+    { category = "ac", func = ip_in_cidr_range, args = { "rules-allowed-cidrs-set", request.ip_decimal_lexical } }
+  }
+
+  return check_rules(rule_categories)
 end
 
-local function increment_partial_hourly_request_counters(unix_time_milliseconds)
-  for i = 1, 60 do
-    local timebucket_in_milliseconds = unix_time_milliseconds + 60000 * (i - 1)
-    local timebucket = get_time_bucket_from_timestamp(timebucket_in_milliseconds, true)
-    local key = wafris_prefix .. "hr-ct:" .. timebucket
-    redis.call("INCR", key)
-    -- Expire the key after 121 minutes if it has no expiry
-    redis.call("EXPIRE", key, 7260)
+local request = {
+  ["ip"] = ARGV[1],
+  ["ip_decimal_lexical"] = string.rep("0", 39 - #ARGV[2]) .. ARGV[2],
+  ["ts_in_milliseconds"] = ARGV[3],
+  ["ts_in_seconds"] = ARGV[3] / 1000,
+  ["user_agent"] = ARGV[4],
+  ["path"] = ARGV[5],
+  ["parameters"] = ARGV[6],
+  ["host"] = ARGV[7],
+  ["method"] = ARGV[8],
+  ["headers"] = ARGV[9],
+  ["post_body"] = ARGV[10],
+  ["ip_id"] = set_property_value_id_lookups("i", ARGV[1]),
+  ["user_agent_id"] = set_property_value_id_lookups("u", ARGV[4]),
+  ["path_id"] = set_property_value_id_lookups("p", ARGV[5]),
+  ["parameters_id"] = set_property_value_id_lookups("a", ARGV[6]),
+  ["host_id"] = set_property_value_id_lookups("h", ARGV[7]),
+  ["method_id"] = set_property_value_id_lookups("m", ARGV[8])
+}
+
+  local current_timebucket = get_timebucket(request.ts_in_seconds)
+  local blocked_rule = false
+  local blocked_category = nil
+  local treatment = "p"
+  local stream_id
+
+  if use_timestamps_as_request_ids == true then
+      stream_id = request.ts_in_milliseconds
+  else
+      stream_id = "*"
   end
-end
 
--- Configuration
-local max_requests = 100000
-local max_requests_per_ip = 10000
-
-local client_ip = ARGV[1]
-local client_ip_to_decimal = ARGV[2]
-local unix_time_milliseconds = ARGV[3]
-local unix_time = ARGV[3] / 1000
-local user_agent = ARGV[4]
-local request_path = ARGV[5]
-local host = ARGV[6]
-
--- Initialize local variables
-local request_id = get_request_id(nil, client_ip, max_requests)
-local current_timebucket = get_time_bucket_from_timestamp(unix_time_milliseconds, false)
-
--- CARD DATA COLLECTION
-increment_partial_hourly_request_counters(unix_time_milliseconds)
-
--- GRAPH DATA COLLECTION
-add_to_graph_timebucket(current_timebucket, request_id)
-
--- LEADERBOARD DATA COLLECTION
-increment_timebucket_for("ip:", current_timebucket, client_ip)
-increment_timebucket_for("ua:", current_timebucket, user_agent)
-increment_timebucket_for("path:", current_timebucket, request_path)
-increment_timebucket_for("host:", current_timebucket, host)
-
--- BLOCKING LOGIC
--- TODO: ZRANGEBYSCORE is deprecated in Redis 6.2+. Replace with ZRANGE
-if
-  -- TODO: When we introduce ranges we'll have to do an exact check followed by a range starting with decimal ip to infinity.
-  -- If the first result returned is "END" that means it falls in the range
-
-  -- ZRANGEBYSCORE will always return a lua table, even if empty
-  -- This call is checking if the table is empty
-  next(redis.call("ZRANGEBYSCORE", "w:blocked-ranges", client_ip_to_decimal, client_ip_to_decimal, "LIMIT", 0, 1))
-  ~= nil
-then
-  increment_timebucket_for("blk:", current_timebucket, client_ip)
+  local stream_args = {
+    "XADD",
+    "rStream",
+    "MINID",
+    tostring((current_timebucket - EXPIRATION_IN_SECONDS) * 1000 ),
+    stream_id,
+    "i", request.ip_id,
+    "u", request.user_agent_id,
+    "p", request.path_id,
+    "h", request.host_id,
+    "m", request.method_id,
+    "a", request.parameters_id,
+  }
+
+  local allowed_rule, allowed_category = check_allowed(request)
+
+  if allowed_rule then
+    table.insert(stream_args, "t")
+    table.insert(stream_args, "a")
+
+    treatment = "a"
+
+    table.insert(stream_args, "ac")
+    table.insert(stream_args, allowed_category)
+
+    table.insert(stream_args, "ar")
+    table.insert(stream_args, allowed_rule)
+  else
+    blocked_rule, blocked_category = check_blocks(request)
+  end
+
+  if blocked_rule then
+    table.insert(stream_args, "t")
+    table.insert(stream_args, "b")
+
+    treatment = "b"
+
+    table.insert(stream_args, "bc")
+    table.insert(stream_args, blocked_category)
+
+    table.insert(stream_args, "br")
+    table.insert(stream_args, blocked_rule)
+  end
+
+  if blocked_rule == false and allowed_rule == false then
+    table.insert(stream_args, "t")
+    table.insert(stream_args, "p")
+  end
+
+  local request_id = redis.call(unpack(stream_args))
+
+  increment_leaderboard_for("i", request.ip_id, current_timebucket)
+  increment_leaderboard_for("u", request.user_agent_id, current_timebucket)
+  increment_leaderboard_for("p", request.path_id, current_timebucket)
+  increment_leaderboard_for("a", request.parameters_id, current_timebucket)
+  increment_leaderboard_for("h", request.host_id, current_timebucket)
+  increment_leaderboard_for("m", request.method_id, current_timebucket)
+  increment_leaderboard_for("t", treatment, current_timebucket)
+
+  set_property_to_requests_list("i", request.ip_id, request_id, current_timebucket)
+  set_property_to_requests_list("u", request.user_agent_id, request_id, current_timebucket)
+  set_property_to_requests_list("p", request.path_id, request_id, current_timebucket)
+  set_property_to_requests_list("a", request.parameters_id, request_id, current_timebucket)
+  set_property_to_requests_list("h", request.host_id, request_id, current_timebucket)
+  set_property_to_requests_list("m", request.method_id, request_id, current_timebucket)
+  set_property_to_requests_list("t", treatment, request_id, current_timebucket)
+
+  if blocked_rule ~= false then
+    increment_leaderboard_for("bc", blocked_category, current_timebucket)
+    set_property_to_requests_list("bc", blocked_category, request_id, current_timebucket)
+
+    increment_leaderboard_for("br", blocked_rule, current_timebucket)
+    set_property_to_requests_list("br", blocked_rule, request_id, current_timebucket)
+  end
+
+  if allowed_rule ~= false then
+    increment_leaderboard_for("ac", allowed_category, current_timebucket)
+    set_property_to_requests_list("ac", allowed_category, request_id, current_timebucket)
+
+    increment_leaderboard_for("ar", allowed_rule, current_timebucket)
+    set_property_to_requests_list("ar", allowed_rule, request_id, current_timebucket)
+  end
+
+if blocked_rule ~= false then
   return "Blocked"
--- No Matches
-else
+elseif allowed_rule ~= false then
   return "Allowed"
+else
+  return "Passed"
 end

data/lib/wafris/configuration.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'version'
+
 module Wafris
   class Configuration
     attr_accessor :redis
@@ -12,7 +14,8 @@ module Wafris
       )
       @redis_pool_size = 20
 
-      set_version if ENV['REDIS_URL']
+      # TODO: update HUB with the REDIS_URL on startup
+      create_settings if ENV['REDIS_URL']
     end
 
     def connection_pool
@@ -33,13 +36,11 @@ module Wafris
       CONNECTION_ERROR
     end
 
-    def set_version
-      version_line = File.open(
-        file_path("wafris_core"),
-        &:readline
-      )
-      version = version_line.slice(/v\d.\d/)
-      redis.set('version', version)
+    def create_settings
+      redis.hset('waf-settings',
+                 'version', Wafris::VERSION,
+                 'client', 'ruby',
+                 'redis-host', 'heroku')
     end
 
     def core_sha

data/lib/wafris/middleware.rb

@@ -7,7 +7,7 @@ module Wafris
     end
 
     def call(env)
-      user_defined_proxies = ENV['MY_PROXIES'].split(',') if ENV['MY_PROXIES']
+      user_defined_proxies = ENV['TRUSTED_PROXY_RANGES'].split(',') if ENV['TRUSTED_PROXY_RANGES']
 
       valid_ipv4_octet = /\.(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/
 

data/lib/wafris/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wafris
-  VERSION = "0.8.5"
+  VERSION = "0.9.1"
 end

data/lib/wafris.rb

@@ -13,7 +13,7 @@ module Wafris
   class << self
     def configure
       yield configuration
-      configuration.set_version
+      configuration.create_settings
     end
 
     def configuration
@@ -24,26 +24,20 @@ module Wafris
       @configuration = Wafris::Configuration.new
     end
 
-    # ip: the IP of the client making the request, may be from x-forwarded-for
-    # user_agent: full user agent making the request
-    # path: path including parameters of the request
-    # host: host (website/domain) making the request
-    # time: UTC time of the request (from the logs to match things up)
-
     def allow_request?(request)
       configuration.connection_pool.with do |conn|
-        time = Time.now.to_f * 1000
-        puts "WAF LOG: headers with http-x-forwarded-for key #{request.get_header(Rack::Request::HTTP_X_FORWARDED_FOR)}"
-        puts "WAF LOG: Client IP #{request.ip}"
+        time = Time.now.utc.to_i * 1000
         status = conn.evalsha(
           configuration.core_sha,
           argv: [
             request.ip,
             IPAddr.new(request.ip).to_i,
-            time.to_i,
+            time,
             request.user_agent,
             request.path,
-            request.host
+            request.params,
+            request.host,
+            request.request_method
           ]
         )
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wafris
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.9.1
 platform: ruby
 authors:
 - Micahel Buckbee
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-22 00:00:00.000000000 Z
+date: 2023-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -186,10 +186,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/lua/dist/wafris_core.lua
-- lib/lua/src/get_time_buckets.lua
-- lib/lua/src/queries.lua
-- lib/lua/src/seeds/data_load.lua
-- lib/lua/src/time_bucket.lua
 - lib/wafris.rb
 - lib/wafris/configuration.rb
 - lib/wafris/middleware.rb
@@ -199,11 +195,8 @@ homepage:
 licenses:
 - Elastic-2.0
 metadata: {}
-post_install_message: |2
-      Thank you for installing the wafris gem. Please note that this is
-      BETA software. We may ask that you clear your redis instance but
-      will do our best to help migrate any block or allow rules that
-      you have created.
+post_install_message: "    Thank you for installing the wafris gem. \n    \n    If
+  you haven't already, please sign up for Wafris Hub at:\n\n    https://wafris.org\n\n"
 rdoc_options: []
 require_paths:
 - lib
@@ -221,5 +214,5 @@ requirements: []
 rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
-summary: Web application firewall for Rack apps
+summary: Web Application Firewall for Rack apps
 test_files: []

data/lib/lua/src/get_time_buckets.lua

@@ -1,58 +0,0 @@
-function get_time_bucket_from_timestamp(unix_time_milliseconds)
-  local function calculate_years_number_of_days(yr)
-    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
-  end
-
-  local function get_year_and_day_number(year, days)
-    while days >= calculate_years_number_of_days(year) do
-      days = days - calculate_years_number_of_days(year)
-      year = year + 1
-    end
-    return year, days
-  end
-
-  local function get_month_and_month_day(days, year)
-    local days_in_each_month = {
-      31,
-      (calculate_years_number_of_days(year) == 366 and 29 or 28),
-      31,
-      30,
-      31,
-      30,
-      31,
-      31,
-      30,
-      31,
-      30,
-      31,
-    }
-
-    for month = 1, #days_in_each_month do
-      if days - days_in_each_month[month] <= 0 then
-        return month, days
-      end
-      days = days - days_in_each_month[month]
-    end
-  end
-
-  local unix_time = unix_time_milliseconds / 1000
-  local year = 1970
-  local days = math.ceil(unix_time / 86400)
-  local month = nil
-
-  year, days = get_year_and_day_number(year, days)
-  month, days = get_month_and_month_day(days, year)
-  local hours = math.floor(unix_time / 3600 % 24)
-  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
-  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
-  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
-end
-
-function get_time_buckets(unix_time_milliseconds)
-  local time_buckets = {}
-
-  for i = 23, 0, -1 do
-    table.insert(time_buckets, get_time_bucket_from_timestamp(unix_time_milliseconds - (1000 * 60 * 60 * i)))
-  end
-  return time_buckets
-end

data/lib/lua/src/queries.lua

@@ -1,14 +0,0 @@
-local function num_requests(start_time, end_time)
-  local request_keys = redis.call('KEYS', 'unique-requests:*')
-  redis.call('PFMERGE', 'merged_unique-requests', unpack(request_keys))
-  return redis.call('PFCOUNT', 'merged_unique-requests')
-end
-
-local function unique_ips(start_time, end_time)
-  local ip_keys = redis.call('KEYS', 'unique-ips:*')
-  redis.call('PFMERGE', 'merged_unique-ips', unpack(ip_keys))
-  return redis.call('PFCOUNT', 'merged_unique-ips')
-end
-
-redis.debug("Request count: ", num_requests(0, 10000000))
-redis.debug("IP request count: ", unique_ips(0, 10000000))

data/lib/lua/src/seeds/data_load.lua

@@ -1,104 +0,0 @@
--- Template strings below are replaced with generated
--- data from the ip_data_generator.rb script
--- local ipArray = { }
--- local timestampArray = { }
--- redis.debug("Timestamp count: ", #timestampArray)
-
-local function get_time_bucket_from_timestamp(unix_time_milliseconds)
-  local function calculate_years_number_of_days(yr)
-    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
-  end
-
-  local function get_year_and_day_number(year, days)
-    while days >= calculate_years_number_of_days(year) do
-      days = days - calculate_years_number_of_days(year)
-      year = year + 1
-    end
-    return year, days
-  end
-
-  local function get_month_and_month_day(days, year)
-    local days_in_each_month = {
-      31,
-      (calculate_years_number_of_days(year) == 366 and 29 or 28),
-      31,
-      30,
-      31,
-      30,
-      31,
-      31,
-      30,
-      31,
-      30,
-      31,
-    }
-
-    for month = 1, #days_in_each_month do
-      if days - days_in_each_month[month] <= 0 then
-        return month, days
-      end
-      days = days - days_in_each_month[month]
-    end
-  end
-
-  local unix_time = unix_time_milliseconds / 1000
-  local year = 1970
-  local days = math.ceil(unix_time / 86400)
-  local month = nil
-
-  year, days = get_year_and_day_number(year, days)
-  month, days = get_month_and_month_day(days, year)
-  local hours = math.floor(unix_time / 3600 % 24)
-  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
-  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
-  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
-end
-
--- For: Relationship of IP to time of Request (Stream)
-local function get_request_id(timestamp, ip, max_requests)
-  timestamp = timestamp or "*"
-  local request_id = redis.call("XADD", "ip-requests-stream", "MAXLEN", "~", max_requests, timestamp, "ip", ip)
-  return request_id
-end
-
-local function add_to_HLL_request_count(timebucket, request_id)
-  redis.call("PFADD", "unique-requests:" .. timebucket, request_id)
-end
-
--- Configuration
-local max_requests = 100000
-local max_requests_per_ip = 10000
-
--- Interior of this for loop is what should go into wafris_core.lua
-for i = 1, #timestampArray do
-  -- Setup
-  local ip = ipArray[math.random(#ipArray)]
-  local timestamp = timestampArray[i]
-
-  local request_id = get_request_id(timestamp, ip, max_requests)
-
-  -- GRAPH DATA COLLECTION
-  local current_timebucket = get_time_bucket_from_timestamp(timestamp)
-  add_to_HLL_request_count(current_timebucket, request_id)
-
-  -- For: Looking up Requests an IP has made (Stream) / time of request
-  local ip_stream_key = "ip-stream:" .. ip
-  local ip_stream_id =
-    redis.call("XADD", ip_stream_key, "MAXLEN", "~", max_requests_per_ip, "*", "request_id", request_id)
-
-  -- For: Precalc of Number of Requests (Key)
-  local requests_count_key = "requests-count:" .. current_timebucket
-  redis.call("INCR", requests_count_key)
-
-  -- For: Precalc of Number of Requests from an IP (Key)
-  local ips_count_bucket_key = "ips-count:" .. ip .. ":" .. current_timebucket
-  redis.call("INCR", ips_count_bucket_key)
-
-  -- For: Precalc of Number of Unique IPs making Requests (HLL)
-  local ips_count_hll_key = "unique-ips:" .. current_timebucket
-  redis.call("PFADD", ips_count_hll_key, ip)
-
-  -- For: Leaderboard of IPs with Request count as score
-  local ip_leaderboard_sset_key = "ip-leader-sset:" .. current_timebucket
-  redis.call("ZINCRBY", ip_leaderboard_sset_key, 1, ip)
-end

data/lib/lua/src/time_bucket.lua

@@ -1,40 +0,0 @@
--- Code was pulled from https://otland.net/threads/how-convert-timestamp-to-date-type.251657/
--- An alternate solution is https://gist.github.com/markuman/e96d04139cd8acc33604
-local function get_time_bucket_from_timestamp(unix_time_milliseconds)
-  local function calculate_years_number_of_days(yr)
-    return (yr % 4 == 0 and (yr % 100 ~= 0 or yr % 400 == 0)) and 366 or 365
-  end
-
-  local function get_year_and_day_number(year, days)
-    while days >= calculate_years_number_of_days(year) do
-      days = days - calculate_years_number_of_days(year)
-      year = year + 1
-    end
-    return year, days
-  end
-
-  local function get_month_and_month_day(days, year)
-    local days_in_each_month = {
-      31,
-      (calculate_years_number_of_days(year) == 366 and 29 or 28),
-      31, 30, 31,30,31,31,30,31,30,31
-    }
-
-    for month = 1, #days_in_each_month do
-      if days - days_in_each_month[month] <= 0 then return month, days end
-      days = days - days_in_each_month[month]
-    end
-  end
-
-  local unix_time = unix_time_milliseconds / 1000
-  local year = 1970
-  local days = math.ceil(unix_time/86400)
-  local month = nil
-
-  year, days = get_year_and_day_number(year, days)
-  month, days = get_month_and_month_day(days, year)
-  local hours = math.floor(unix_time / 3600 % 24)
-  -- local minutes, seconds = math.floor(unix_time / 60 % 60), math.floor(unix_time % 60)
-  -- hours = hours > 12 and hours - 12 or hours == 0 and 12 or hours
-  return string.format("%04d-%02d-%02d-%02d", year, month, days, hours)
-end