ronin-db 0.1.0.beta1

data/lib/ronin/db/cli/commands/urls.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/model_command'
+require 'ronin/db/cli/modifiable'
+
+module Ronin
+  module DB
+    class CLI
+      module Commands
+        #
+        # Manages all URLs in the database.
+        #
+        # ## Usage
+        #
+        #     ronin-db urls [options]
+        #
+        # ## Options
+        #
+        #         --db NAME                    The database to connect to (Default: default)
+        #         --db-uri URI                 The database URI to connect to
+        #     -v, --verbose                    Enables verbose output
+        #         --add VALUE                  Adds a value to the database
+        #         --import FILE                Imports the values from the FILE into the database
+        #         --delete VALUE               Deletes a value from the database
+        #         --delete-all                 Deletes all values from the database
+        #          --http
+        #          --https
+        #      -H, --host HOST
+        #      -P, --port PORT
+        #      -d, --directory SUBDIR
+        #      -E, --ext EXT
+        #      -q NAME
+        #          --with-query-param
+        #      -Q VALUE
+        #          --with-query-value
+        #
+        class Urls < ModelCommand
+
+          include Modifiable
+
+          model_file 'ronin/db/url'
+          model_name 'URL'
+
+          option :http, desc: 'Searches for http:// URLs' do
+            @query_method_calls << :http
+          end
+
+          option :https, desc: 'Searches for https:// URLs' do
+            @query_method_calls << :https
+          end
+
+          option :host, short: '-H',
+                        value: {
+                          type: String,
+                          usage: 'HOST'
+                        },
+                        desc: 'Searches for the associated HOST(s)' do |host|
+                          @query_method_calls << [:with_host_name, [host]]
+                        end
+
+          option :port, short: '-p',
+                        value: {
+                          type: Integer,
+                          usage: 'PORT'
+                        },
+                        desc: 'Searches for the associated PORT(s)' do |port|
+                          @query_method_calls << [:with_port_number, [port]]
+                        end
+
+          option :path, value: {
+                          type: String,
+                          usage: 'PATH'
+                        },
+                        desc: 'Searches for all URLs with the PATH' do |path|
+                          @query_method_calls << [:where, [], {path: path}]
+                        end
+
+          option :fragment, value: {
+                              type: String,
+                              usage: 'FRAGMENT'
+                            },
+                            desc: 'Searches for all URLs with the FRAGMENT' do |fragment|
+                              @query_method_calls << [:where, [], {fragment: fragment}]
+                            end
+
+          option :directory, short: '-d',
+                             value: {
+                               type:  String,
+                               usage: 'DIR'
+                             },
+                             desc: 'Searches for the associated DIR' do |dir|
+                               @query_method_calls << [:with_directory, [dir]]
+                             end
+
+          option :with_ext, short: '-E',
+                            value: {
+                              type:  String,
+                              usage: 'EXT'
+                            },
+                            desc: 'Searches for URLs with the file extension' do |ext|
+                              @query_method_calls << [:with_ext, [ext]]
+                            end
+
+          option :query_string, short: '-q',
+                                value: {
+                                  type:  String,
+                                  usage: 'STRING',
+                                },
+                                desc: 'Searches for all URLs with the query string' do |string|
+                                  @query_method_calls << [:where, [], {query: string}]
+                                end
+
+          option :with_query_param, short: '-P',
+                                    value: {
+                                      type: String,
+                                      usage: 'NAME[=VALUE]'
+                                    },
+                                    desc: 'Searches for the associated query-param NAME(s)' do |string|
+                                      name, value = string.split('=',2)
+
+                                      if value
+                                        @query_method_calls << [:with_query_param, [name, value]]
+                                      else
+                                        @query_method_calls << [:with_query_param_name, [name]]
+                                      end
+                                    end
+
+          option :with_query_param_name, value: {
+                                           type:  String,
+                                           usage: 'NAME'
+                                         },
+                                         desc: 'Searches for the associated query-param VALUE(s)' do |name|
+                                           @query_method_calls << [:with_query_param_name, [name]]
+                                         end
+
+          option :with_query_param_value, value: {
+                                            type:  String,
+                                            usage: 'VALUE'
+                                          },
+                                          desc: 'Searches for the associated query-param VALUE(s)' do |value|
+                                            @query_method_calls << [:with_query_param_value, [value]]
+                                          end
+
+          description 'Manages URLs'
+
+          man_page 'ronin-db-urls.1'
+
+        end
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/database_command.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/command'
+require 'ronin/db/cli/uri_methods'
+require 'ronin/db'
+
+module Ronin
+  module DB
+    class CLI
+      #
+      # Base class for all commands that access the database.
+      #
+      class DatabaseCommand < Command
+
+        include URIMethods
+
+        option :db, value: {
+                      type:    DB.config.keys,
+                      default: :default,
+                      usage:   'NAME'
+                    },
+                    desc: 'The database to connect to'
+
+        option :db_uri, value: {
+                          type:  String,
+                          usage: 'URI'
+                        },
+                        desc: 'The database URI to connect to'
+
+        #
+        # The database connection configuration.
+        #
+        # @return [Hash{Symbol => String,Integer}]
+        #
+        def config
+          if options[:db_uri]
+            parse_uri(options[:db_uri])
+          else
+            DB.config[options[:db]]
+          end
+        end
+
+        #
+        # Connects to the database.
+        #
+        def connect
+          DB.connect(config)
+        end
+
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/model_command.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/database_command'
+require 'ronin/core/cli/logging'
+
+require 'command_kit/options/verbose'
+
+module Ronin
+  module DB
+    class CLI
+      #
+      # A base-command for database models commands.
+      #
+      class ModelCommand < DatabaseCommand
+
+        include CommandKit::Options::Verbose
+        include Core::CLI::Logging
+
+        #
+        # Sets or gets the model file to require.
+        #
+        # @param [String, nil] new_model_file
+        #   The new model file.
+        #
+        # @return [String]
+        #   The model file to require.
+        #
+        # @raise [NotImplementedError]
+        #   The class did not define a `model_file`.
+        #
+        # @example
+        #   model_file 'ronin/db/foo'
+        #
+        def self.model_file(new_model_file=nil)
+          if new_model_file
+            @model_file = new_model_file
+          else
+            @model_file ||= if superclass < ModelCommand
+                              superclass.model_file
+                            else
+                              raise(NotImplementedError,"#{self} did not define model_file")
+                            end
+          end
+        end
+
+        #
+        # Sets or gets the model name to lookup.
+        #
+        # @param [String, nil] new_model_name
+        #   The new model name.
+        #
+        # @return [String]
+        #   The model name to lookup.
+        #
+        # @raise [NotImplementedError]
+        #   The class did not define a `model_name`.
+        #
+        # @example
+        #   model_name 'Foo'
+        #
+        def self.model_name(new_model_name=nil)
+          if new_model_name
+            @model_name = new_model_name
+          else
+            @model_name ||= if superclass < ModelCommand
+                              superclass.model_name
+                            else
+                              raise(NotImplementedError,"#{self} did not define model_name")
+                            end
+          end
+        end
+
+        # The query method calls to chain together.
+        #
+        # @return [Array<(Symbol),
+        #                (Symbol, Array),
+        #                (Symbol, Hash),
+        #                (Symbol, Array, Hash)>]
+        attr_reader :query_method_calls
+
+        #
+        # Initializes the command.
+        #
+        # @param [Hash{Symbol => Object}] kwargs
+        #   Additional keyword arguments.
+        #
+        def initialize(**kwargs)
+          super(**kwargs)
+
+          @query_method_calls = []
+        end
+
+        #
+        # Runs the command.
+        #
+        def run
+          connect
+          list
+        end
+
+        #
+        # Connects to the database.
+        #
+        def connect
+          # connect to the database but do not load other models.
+          DB.connect(config, load_models: false)
+
+          # load and connect the model
+          model.connection
+        end
+
+        #
+        # The model to query.
+        #
+        # @return [Class<ActiveRecord::Base>]
+        #   The loaded model.
+        #
+        # @raise [LoadError]
+        #   The {model_file} could not be loaded.
+        #
+        # @raise [NameError]
+        #   The {model_name} was not found within {Ronin::DB}.
+        #
+        def load_model
+          require self.class.model_file
+
+          Ronin::DB.const_get(self.class.model_name)
+        end
+
+        #
+        # The model to query.
+        #
+        # @return [Class<ActiveRecord::Base>]
+        #   The loaded model.
+        #
+        def model
+          @model ||= load_model
+        end
+
+        #
+        # Queries and lists records.
+        #
+        def list
+          records = query
+          records.each(&method(:print_record))
+        end
+
+        #
+        # Builds a new query by chaining together the method calls defined by
+        # {#query_method_calls}.
+        #
+        # @return [ActiveRecord::Relation, ActiveRecord::QueryMethods::WhereChain]
+        #   The new query.
+        #
+        def query
+          common_object_methods = Object.public_instance_methods
+
+          query = model.all
+
+          @query_method_calls.each do |method,arguments,kwargs={}|
+            if common_object_methods.include?(method)
+              raise(ArgumentError,"cannot call method Object##{method} on query #{query.inspect}")
+            end
+
+            query = query.public_send(method,*arguments,**kwargs)
+          end
+
+          return query
+        end
+
+        #
+        # Prints the given record.
+        #
+        # @param [ActiveRecord::Base] record
+        #   The record to print.
+        #
+        def print_record(record)
+          puts record
+        end
+
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/modifiable.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+module Ronin
+  module DB
+    class CLI
+      #
+      # Allows a {ModelCommand} to add, import, delete, or delete all records.
+      #
+      module Modifiable
+        #
+        # Adds the `--add`, `--import`, `--delete`, and `--delete-all` options
+        # to the command.
+        #
+        # @param [Class<ModelCommand>] command
+        #   The command class including {Modifiable}.
+        #
+        def self.included(command)
+          command.option :add, value: {
+                                 type: String,
+                                 usage: 'VALUE'
+                               },
+                               desc: 'Adds a value to the database'
+
+          command.option :import, value: {
+                                    type:  String,
+                                    usage: 'FILE'
+                                  },
+                                  desc: 'Imports the values from the FILE into the database'
+
+          command.option :delete, value: {
+                                    type: String,
+                                    usage: 'VALUE'
+                                  },
+                                  desc: 'Deletes a value from the database'
+
+          command.option :delete_all, desc: 'Deletes all values from the database'
+        end
+
+        #
+        # Runs the command.
+        #
+        def run
+          connect
+
+          if options[:add]
+            add(options[:add])
+          elsif options[:import]
+            import_file(options[:import])
+          elsif options[:delete]
+            delete(options[:delete])
+          elsif options[:delete_all]
+            delete_all
+          else
+            list
+          end
+        end
+
+        #
+        # Adds a value to the database.
+        #
+        # @param [String] value
+        #   The value to add.
+        #
+        def add(value)
+          record = model.import(value)
+
+          unless record.valid?
+            print_error "failed to import #{value}!"
+
+            record.errors.full_messages.each do |message|
+              print_error " - #{message}"
+            end
+          end
+        end
+
+        #
+        # Imports the values from the givne file.
+        #
+        # @param [String] path
+        #   The path to the file.
+        #
+        def import_file(path)
+          unless File.file?(path)
+            print_error "no such file or directory: #{path}"
+            exit(-1)
+          end
+
+          File.open(path) do |file|
+            model.transaction do
+              file.each_line(chomp: true) do |value|
+                log_info "Importing #{value} ..." if verbose?
+
+                add(value)
+              end
+            end
+          end
+        end
+
+        #
+        # Deletes a value from the database.
+        #
+        # @param [String] value
+        #   The value to lookup and delete.
+        #
+        def delete(value)
+          if (record = model.lookup(value))
+            record.destroy
+          else
+            print_error "value does not exist in the database: #{value}"
+            exit(-1)
+          end
+        end
+
+        #
+        # Deletes all values from the database.
+        #
+        def delete_all
+          model.destroy_all
+        end
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/resources_command.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/ui/cli/model_command'
+require 'ronin/db/model/importable'
+
+require 'dm-serializer'
+
+module Ronin
+  module DB
+    module CLI
+      #
+      # A base-command class for listing Database Resources.
+      #
+      class ResourcesCommand < ModelCommand
+
+        option :csv, type:        true,
+                     description: 'CSV output'
+
+        option :xml, type:        true,
+                     description: 'XML output'
+
+        option :yaml, type:        true,
+                      description: 'YAML output'
+
+        option :json, type:        true,
+                      description: 'JSON output'
+
+        #
+        # Default method performs the query and prints the found resources.
+        #
+        # @since 1.1.0
+        #
+        # @api semipublic
+        #
+        def execute
+          if @import
+            self.class.model.import(@import) do |resource|
+              print_info "Imported #{resource}"
+            end
+          else
+            print_resources(query)
+          end
+        end
+
+        protected
+
+        #
+        # Sets the model used by the command.
+        #
+        # @see ModelCommand.model
+        #
+        # @since 1.3.0
+        #
+        def self.model(model=nil)
+          if (model && model < Model::Importable)
+            option :import, type:        String,
+                            flag:        '-i',
+                            usage:       'FILE',
+                            description: 'The file to import'
+          end
+
+          return super(model)
+        end
+
+        #
+        # Default method which will print every queried resource.
+        #
+        # @param [DataMapper::Resource] resource
+        #   A queried resource from the Database.
+        #
+        # @since 1.1.0
+        #
+        # @api semipublic
+        #
+        def print_resource(resource)
+          puts resource
+        end
+
+        #
+        # Prints multiple resources.
+        #
+        # @param [DataMapper::Collection] resources
+        #   The query to print.
+        #
+        # @since 1.1.0
+        #
+        # @api semipublic
+        #
+        def print_resources(resources)
+          if    csv?  then puts resources.to_csv
+          elsif xml?  then puts resources.to_xml
+          elsif yaml? then puts resources.to_yaml
+          elsif json? then puts resources.to_json
+          else
+            resources.each { |resource| print_resource(resource) }
+          end
+        end
+
+      end
+    end
+  end
+end