bqm 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d78da6d030479d62b8fc734bde08fddc8ff97ca4502855287cb543b1d5f50fd
-  data.tar.gz: 836675f08cab5fe97744eae3c4f53182e2bbc9928f099540e94c36af576f3b56
+  metadata.gz: 1a14c962fd0bf2a8c456bf7b9ff5416036289f61a8d4030ed246a77b6115aec0
+  data.tar.gz: d9d0bf051756b2eba968cdb37945fd953db2746b646f6a9888c4100207979792
 SHA512:
-  metadata.gz: 8d3f43f194866b90855ec27c36dfecdd3ba5131b424616f08eaf7ac0f75c02975d92553ae02f0f807246cfe61fd198ee2da06f6068b0e3d0e9c284f57ec41eca
-  data.tar.gz: fa5e9708365a8ac529c0a67b8c46bbc3aa901c91dac6d01b706d390c1211a534887eedcfaf31cfcbe0fd3350e4f5d9b1d1047fb6303e1543c7a4ec38d9b9079b
+  metadata.gz: 2283d7f843cfec5c8de3cf92044a12ed3731474125c6fa70675476cbce028b6dc26d8bdfd6ceafd8dd940d2a33a7d763558834a7ecbc709ab72b1ebcba18dc0c
+  data.tar.gz: 5f22350ef473520e98f26facab2a7a583b07739f0aa03b9d0fb312199a696e074ae311191476883af12bd46b1bf2413e4c519dd997a257d9ddca52e022ce4222

data/DOC.md

@@ -0,0 +1,4 @@
+## Options
+
+- `-i`, `--local-sets`: if several items are provided, they must be comma (`,`) separated. Items can be either a Bloodhound custom query file, a bqm or a folder. In case of a bqm query sets file (similar to `query-sets.json`) or a folder, bqm will try to parse all JSON files inside it, so the folder should not contain other types of JSON files.
+- `--ignore-default`: ignore the default `query-sets.json`. It's useful for **offline** usage or if you don't want the default data sets.

data/bin/bqm

@@ -14,13 +14,42 @@ def find_dataset
   raise IOError, "The dataset file #{source_file} does not exist or is unreadable."
 end
 
-def merge(source)
+# Merge remote sets defined in data/query-sets.json
+def merge_remote(source, verbose: false)
   sets = get_datasets(source)
   queries = []
   sets.each do |s|
     customqueries = Net::HTTP.get(URI(s))
     data = JSON.parse(customqueries)
     queries += data['queries']
+    puts "  [*] File merged: #{s}" if verbose
+  end
+  queries
+end
+
+# Merge local sets provided by the user
+def merge_local(sources, verbose: false)
+  queries = []
+  sources.each do |source|
+    # If it's a file parse it
+    if File.file?(source) && File.readable?(source)
+      data = json_load(source)
+      if data['queries']
+        queries += data['queries']
+      elsif data['sets']
+        queries += merge_remote(source, verbose: verbose)
+      else
+        raise KeyError, "The file #{source} is neiter a Bloodhound custom query file nor a bqm query sets file"
+      end
+      puts "  [*] File merged: #{source}" if verbose
+    # If it's a folder, retrieve all JSON files and assumes there are queries files
+    # Then recursive call for a normal file parsing
+    elsif File.directory?(source) && File.readable?(source)
+      qfiles = Dir.glob('*.json', base: source).map { |f| File.absolute_path(f, source) }
+      queries += merge_local(qfiles, verbose: verbose)
+    else
+      raise IOError, "The dataset file/directory #{source} does not exist or is unreadable."
+    end
   end
   queries
 end
@@ -59,27 +88,40 @@ def pretty_link(lst)
 end
 
 def get_datasets(source)
+  src = json_load(source)
+  src['sets']
+end
+
+def json_load(file)
   # ruby 3.0+
   begin
-    src = JSON.load_file(source)
+    src = JSON.load_file(file)
   rescue NoMethodError # ruby 2.7 retro-compatibility
-    src = JSON.parse(File.read(source))
+    src = JSON.parse(File.read(file))
   end
-  src['sets']
+  src
 end
 
 if __FILE__ == $PROGRAM_NAME
   source = find_dataset
 
   require 'optparse'
-  options = {}
+  options = {
+    'local-sets': []
+  }
   OptionParser.new do |parser|
     parser.banner = 'Usage: bqm [options]'
 
     parser.on('-o', '--output-path PATH', 'Path where to store the query file')
     parser.on('-l', '--list', 'List available datasets')
+    parser.on('-i', '--local-sets FILE,DIRECTORY,...', Array, 'Local custom queries files/directories') do |f|
+      options[:'local-sets'] += f
+    end
+    parser.on('--ignore-default', 'Ignore the default query-sets.json')
+    parser.on('-v', '--verbose', 'Display the name of the merged files/sets')
     parser.separator ''
     parser.separator 'Example: bqm -o ~/.config/bloodhound/customqueries.json'
+    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/home/user/folder'
   end.parse!(into: options)
 
   out = options[:'output-path']
@@ -95,15 +137,18 @@ if __FILE__ == $PROGRAM_NAME
     if File.file?(out) && File.readable?(out)
       puts "[+] The output path #{out} already exists"
       puts '[?] Do you want to overwrite it? [y/n]'
-      if gets.chomp == 'y'
+      if $stdin.gets.chomp == 'y'
         puts '[?] What to do with the existing queries? (merge / discard) [m/d]'
-        flags[:merge_actual] = true if gets.chomp == 'm'
+        flags[:merge_actual] = true if $stdin.gets.chomp == 'm'
       else
         exit
       end
     end
     puts '[+] Fetching and merging datasets'
-    data = merge(source)
+    data = []
+    data = merge_remote(source, verbose: options[:verbose]) unless options[:'ignore-default']
+    local_set = options[:'local-sets']
+    data += merge_local(local_set, verbose: options[:verbose]) if local_set
     if flags[:merge_actual]
       puts '[+] Merging your existing queries'
       data += JSON.parse(File.read(out))['queries']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bqm
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Alexandre ZANNI
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-27 00:00:00.000000000 Z
+date: 2023-06-29 00:00:00.000000000 Z
 dependencies: []
 description: Deduplicate custom BloudHound queries from different datasets and merge
   them in one customqueries.json file.
@@ -18,6 +18,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- DOC.md
 - LICENSE
 - bin/bqm
 - data/query-sets.json
@@ -26,6 +27,8 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/Acceis/bqm/issues
+  changelog_uri: https://github.com/Acceis/bqm/releases
+  documentation_uri: https://acceis.github.io/bqm/
   homepage_uri: https://github.com/Acceis/bqm
   source_code_uri: https://github.com/Acceis/bqm/
   rubygems_mfa_required: 'true'