bqm 1.4.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a7ea69f913278933cdcbae6a764b166d7990931ce1f68c1264f3c0127f548ad
-  data.tar.gz: 65a2b6a2aa1e78864e48671a2f8b5e4ce395fa0eaead13ce3098806cc6391ce9
+  metadata.gz: 851357946e6ed09933678c74958585dcb137314743d02de03b9f5a56cec6750d
+  data.tar.gz: 05a321f96dcfd330d81869eed88f7ad3b42103bb32b31858a47860b617c376da
 SHA512:
-  metadata.gz: be95aaeccf71fb47aac6179e3be127d5d456fc88e4f67b1798b46dfafcf24cba0220cc6c5569fba98acbd24b9ca2f6bde71adb2604e9b0d9e2e4afeb56bb743f
-  data.tar.gz: c752dc0e0d790b2b9e5337d5622cf05fd5d553558d7c3cb6654829ad54d8fb88157849e4d70c5a55589327c136dd2d442a857089d244923062953ba6b3a06f83
+  metadata.gz: 025c0a65e18b7a7d520f2ad9137dc76200653619b258f8558eee15ebfabc38222da3681d2aff5add7c5ed6e3454ed78c0f18ac6631e69fc2dda30b4a1ca9bbbc
+  data.tar.gz: 5e999b124e978527d8c99bbf551803346ff827965d2d3c4d442c235ab879e08cd517821bcc8c486099c5331abf36543df13d6ebd06b27755bb6563a7412efb63

data/DOC.md

@@ -0,0 +1,4 @@
+## Options
+
+- `-i`, `--local-sets`: if several items are provided, they must be comma (`,`) separated. Items can be either a Bloodhound custom query file, a bqm or a folder. In case of a bqm query sets file (similar to `query-sets.json`) or a folder, bqm will try to parse all JSON files inside it, so the folder should not contain other types of JSON files.
+- `--ignore-default`: ignore the default `query-sets.json`. It's useful for **offline** usage or if you don't want the default data sets.

data/bin/bqm

@@ -15,30 +15,43 @@ def find_dataset
 end
 
 # Merge remote sets defined in data/query-sets.json
-def merge_remote(source)
+def merge_remote(source, verbose: false)
   sets = get_datasets(source)
   queries = []
   sets.each do |s|
     customqueries = Net::HTTP.get(URI(s))
     data = JSON.parse(customqueries)
     queries += data['queries']
+    puts "  [*] File merged: #{s}" if verbose
+  rescue JSON::ParserError
+    # Handle the JSON parsing error
+    puts "  [!] JSON parsing error for #{s}"
   end
   queries
 end
 
 # Merge local sets provided by the user
-def merge_local(sources)
+def merge_local(sources, verbose: false)
   queries = []
   sources.each do |source|
+    # If it's a file parse it
     if File.file?(source) && File.readable?(source)
-      begin
-        data = JSON.load_file(source)
-      rescue NoMethodError # ruby 2.7 retro-compatibility
-        data = JSON.parse(File.read(source))
+      data = json_load(source)
+      if data['queries']
+        queries += data['queries']
+      elsif data['sets']
+        queries += merge_remote(source, verbose: verbose)
+      else
+        raise KeyError, "The file #{source} is neiter a Bloodhound custom query file nor a bqm query sets file"
       end
-      queries += data['queries']
+      puts "  [*] File merged: #{source}" if verbose
+    # If it's a folder, retrieve all JSON files and assumes there are queries files
+    # Then recursive call for a normal file parsing
+    elsif File.directory?(source) && File.readable?(source)
+      qfiles = Dir.glob('*.json', base: source).map { |f| File.absolute_path(f, source) }
+      queries += merge_local(qfiles, verbose: verbose)
     else
-      raise IOError, "The dataset file #{source} does not exist or is unreadable."
+      raise IOError, "The dataset file/directory #{source} does not exist or is unreadable."
     end
   end
   queries
@@ -78,13 +91,18 @@ def pretty_link(lst)
 end
 
 def get_datasets(source)
+  src = json_load(source)
+  src['sets']
+end
+
+def json_load(file)
   # ruby 3.0+
   begin
-    src = JSON.load_file(source)
+    src = JSON.load_file(file)
   rescue NoMethodError # ruby 2.7 retro-compatibility
-    src = JSON.parse(File.read(source))
+    src = JSON.parse(File.read(file))
   end
-  src['sets']
+  src
 end
 
 if __FILE__ == $PROGRAM_NAME
@@ -92,19 +110,21 @@ if __FILE__ == $PROGRAM_NAME
 
   require 'optparse'
   options = {
-    :'local-sets' => []
+    'local-sets': []
   }
   OptionParser.new do |parser|
     parser.banner = 'Usage: bqm [options]'
 
     parser.on('-o', '--output-path PATH', 'Path where to store the query file')
     parser.on('-l', '--list', 'List available datasets')
-    parser.on('-i', '--local-sets FILE,...', Array, 'Local custom queries files') do |f|
+    parser.on('-i', '--local-sets FILE,DIRECTORY,...', Array, 'Local custom queries files/directories') do |f|
       options[:'local-sets'] += f
     end
+    parser.on('--ignore-default', 'Ignore the default query-sets.json')
+    parser.on('-v', '--verbose', 'Display the name of the merged files/sets')
     parser.separator ''
     parser.separator 'Example: bqm -o ~/.config/bloodhound/customqueries.json'
-    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json'
+    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/home/user/folder'
   end.parse!(into: options)
 
   out = options[:'output-path']
@@ -120,20 +140,19 @@ if __FILE__ == $PROGRAM_NAME
     if File.file?(out) && File.readable?(out)
       puts "[+] The output path #{out} already exists"
       puts '[?] Do you want to overwrite it? [y/n]'
-      if STDIN.gets.chomp == 'y'
+      if $stdin.gets.chomp == 'y'
         puts '[?] What to do with the existing queries? (merge / discard) [m/d]'
-        flags[:merge_actual] = true if STDIN.gets.chomp == 'm'
+        flags[:merge_actual] = true if $stdin.gets.chomp == 'm'
       else
         exit
       end
     end
     puts '[+] Fetching and merging datasets'
-    data = merge_remote(source)
+    data = []
+    data = merge_remote(source, verbose: options[:verbose]) unless options[:'ignore-default']
     local_set = options[:'local-sets']
-    if local_set
-      data += merge_local(local_set)
-    end
-    if flags[:'merge_actual']
+    data += merge_local(local_set, verbose: options[:verbose]) if local_set
+    if flags[:merge_actual]
       puts '[+] Merging your existing queries'
       data += JSON.parse(File.read(out))['queries']
     end

data/data/query-sets.json

@@ -12,6 +12,7 @@
     "https://raw.githubusercontent.com/egypt/customqueries/master/customqueries.json",
     "https://raw.githubusercontent.com/trustedsec/CrackHound/main/customqueries.json",
     "https://raw.githubusercontent.com/aress31/bloodhound-utils/main/customqueries.json",
-    "https://raw.githubusercontent.com/ThePorgs/Exegol-images/main/sources/bloodhound/customqueries.json"
+    "https://raw.githubusercontent.com/ThePorgs/Exegol-images/main/sources/assets/bloodhound/customqueries.json"
   ]
 }
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bqm
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Alexandre ZANNI
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-26 00:00:00.000000000 Z
+date: 2023-09-06 00:00:00.000000000 Z
 dependencies: []
 description: Deduplicate custom BloudHound queries from different datasets and merge
   them in one customqueries.json file.
@@ -18,6 +18,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- DOC.md
 - LICENSE
 - bin/bqm
 - data/query-sets.json