bqm 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a7ea69f913278933cdcbae6a764b166d7990931ce1f68c1264f3c0127f548ad
-  data.tar.gz: 65a2b6a2aa1e78864e48671a2f8b5e4ce395fa0eaead13ce3098806cc6391ce9
+  metadata.gz: 1a14c962fd0bf2a8c456bf7b9ff5416036289f61a8d4030ed246a77b6115aec0
+  data.tar.gz: d9d0bf051756b2eba968cdb37945fd953db2746b646f6a9888c4100207979792
 SHA512:
-  metadata.gz: be95aaeccf71fb47aac6179e3be127d5d456fc88e4f67b1798b46dfafcf24cba0220cc6c5569fba98acbd24b9ca2f6bde71adb2604e9b0d9e2e4afeb56bb743f
-  data.tar.gz: c752dc0e0d790b2b9e5337d5622cf05fd5d553558d7c3cb6654829ad54d8fb88157849e4d70c5a55589327c136dd2d442a857089d244923062953ba6b3a06f83
+  metadata.gz: 2283d7f843cfec5c8de3cf92044a12ed3731474125c6fa70675476cbce028b6dc26d8bdfd6ceafd8dd940d2a33a7d763558834a7ecbc709ab72b1ebcba18dc0c
+  data.tar.gz: 5f22350ef473520e98f26facab2a7a583b07739f0aa03b9d0fb312199a696e074ae311191476883af12bd46b1bf2413e4c519dd997a257d9ddca52e022ce4222

data/DOC.md

@@ -0,0 +1,4 @@
+## Options
+
+- `-i`, `--local-sets`: if several items are provided, they must be comma (`,`) separated. Items can be either a Bloodhound custom query file, a bqm or a folder. In case of a bqm query sets file (similar to `query-sets.json`) or a folder, bqm will try to parse all JSON files inside it, so the folder should not contain other types of JSON files.
+- `--ignore-default`: ignore the default `query-sets.json`. It's useful for **offline** usage or if you don't want the default data sets.

data/bin/bqm

@@ -15,30 +15,40 @@ def find_dataset
 end
 
 # Merge remote sets defined in data/query-sets.json
-def merge_remote(source)
+def merge_remote(source, verbose: false)
   sets = get_datasets(source)
   queries = []
   sets.each do |s|
     customqueries = Net::HTTP.get(URI(s))
     data = JSON.parse(customqueries)
     queries += data['queries']
+    puts "  [*] File merged: #{s}" if verbose
   end
   queries
 end
 
 # Merge local sets provided by the user
-def merge_local(sources)
+def merge_local(sources, verbose: false)
   queries = []
   sources.each do |source|
+    # If it's a file parse it
     if File.file?(source) && File.readable?(source)
-      begin
-        data = JSON.load_file(source)
-      rescue NoMethodError # ruby 2.7 retro-compatibility
-        data = JSON.parse(File.read(source))
+      data = json_load(source)
+      if data['queries']
+        queries += data['queries']
+      elsif data['sets']
+        queries += merge_remote(source, verbose: verbose)
+      else
+        raise KeyError, "The file #{source} is neiter a Bloodhound custom query file nor a bqm query sets file"
       end
-      queries += data['queries']
+      puts "  [*] File merged: #{source}" if verbose
+    # If it's a folder, retrieve all JSON files and assumes there are queries files
+    # Then recursive call for a normal file parsing
+    elsif File.directory?(source) && File.readable?(source)
+      qfiles = Dir.glob('*.json', base: source).map { |f| File.absolute_path(f, source) }
+      queries += merge_local(qfiles, verbose: verbose)
     else
-      raise IOError, "The dataset file #{source} does not exist or is unreadable."
+      raise IOError, "The dataset file/directory #{source} does not exist or is unreadable."
     end
   end
   queries
@@ -78,13 +88,18 @@ def pretty_link(lst)
 end
 
 def get_datasets(source)
+  src = json_load(source)
+  src['sets']
+end
+
+def json_load(file)
   # ruby 3.0+
   begin
-    src = JSON.load_file(source)
+    src = JSON.load_file(file)
   rescue NoMethodError # ruby 2.7 retro-compatibility
-    src = JSON.parse(File.read(source))
+    src = JSON.parse(File.read(file))
   end
-  src['sets']
+  src
 end
 
 if __FILE__ == $PROGRAM_NAME
@@ -92,19 +107,21 @@ if __FILE__ == $PROGRAM_NAME
 
   require 'optparse'
   options = {
-    :'local-sets' => []
+    'local-sets': []
   }
   OptionParser.new do |parser|
     parser.banner = 'Usage: bqm [options]'
 
     parser.on('-o', '--output-path PATH', 'Path where to store the query file')
     parser.on('-l', '--list', 'List available datasets')
-    parser.on('-i', '--local-sets FILE,...', Array, 'Local custom queries files') do |f|
+    parser.on('-i', '--local-sets FILE,DIRECTORY,...', Array, 'Local custom queries files/directories') do |f|
       options[:'local-sets'] += f
     end
+    parser.on('--ignore-default', 'Ignore the default query-sets.json')
+    parser.on('-v', '--verbose', 'Display the name of the merged files/sets')
     parser.separator ''
     parser.separator 'Example: bqm -o ~/.config/bloodhound/customqueries.json'
-    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json'
+    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/home/user/folder'
   end.parse!(into: options)
 
   out = options[:'output-path']
@@ -120,20 +137,19 @@ if __FILE__ == $PROGRAM_NAME
     if File.file?(out) && File.readable?(out)
       puts "[+] The output path #{out} already exists"
       puts '[?] Do you want to overwrite it? [y/n]'
-      if STDIN.gets.chomp == 'y'
+      if $stdin.gets.chomp == 'y'
         puts '[?] What to do with the existing queries? (merge / discard) [m/d]'
-        flags[:merge_actual] = true if STDIN.gets.chomp == 'm'
+        flags[:merge_actual] = true if $stdin.gets.chomp == 'm'
       else
         exit
       end
     end
     puts '[+] Fetching and merging datasets'
-    data = merge_remote(source)
+    data = []
+    data = merge_remote(source, verbose: options[:verbose]) unless options[:'ignore-default']
     local_set = options[:'local-sets']
-    if local_set
-      data += merge_local(local_set)
-    end
-    if flags[:'merge_actual']
+    data += merge_local(local_set, verbose: options[:verbose]) if local_set
+    if flags[:merge_actual]
       puts '[+] Merging your existing queries'
       data += JSON.parse(File.read(out))['queries']
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bqm
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Alexandre ZANNI
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-26 00:00:00.000000000 Z
+date: 2023-06-29 00:00:00.000000000 Z
 dependencies: []
 description: Deduplicate custom BloudHound queries from different datasets and merge
   them in one customqueries.json file.
@@ -18,6 +18,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- DOC.md
 - LICENSE
 - bin/bqm
 - data/query-sets.json