ronin-db 0.1.0.beta1

data/gemspec.yml

@@ -0,0 +1,46 @@
+name: ronin-db
+summary: A common database library for managing and querying security data
+description: |
+  ronin-db is a database library for managing and querying security data.
+  ronin-db provides common ORM models for interacting with the database's SQL
+  tables and querying/storing security data, such as URLs, email addresses,
+  host names, IPs, ports, etc. ronin-db also provides CLI commands for managing
+  the database(s).
+
+license: LGPL-3.0
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://ronin-rb.dev
+has_yard: true
+
+generated_files:
+ - man/ronin-db-add.1
+ - man/ronin-db-asn.1
+ - man/ronin-db-creds.1
+ - man/ronin-db-emails.1
+ - man/ronin-db-hosts.1
+ - man/ronin-db-ips.1
+ - man/ronin-db-irb.1
+ - man/ronin-db-edit.1
+ - man/ronin-db-irb.1
+ - man/ronin-db-list.1
+ - man/ronin-db-migrate.1
+ - man/ronin-db-remove.1
+ - man/ronin-db-urls.1
+
+required_ruby_version: ">= 3.0.0"
+
+ruby_dependencies:
+  sqlite: ~> 1.0
+
+jruby_dependencies:
+  activerecord-jdbcsqlite3-adapter: ~> 70.0.pre
+
+dependencies:
+  # Ronin dependencies:
+  ronin-db-activerecord: ~> 0.1.0.beta1
+  ronin-support: ~> 1.0.0.beta1
+  ronin-core: ~> 0.1.0.beta1
+
+development_dependencies:
+  bundler: ~> 2.0

data/lib/ronin/db/cli/command.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/root'
+
+require 'ronin/core/cli/command'
+
+module Ronin
+  module DB
+    class CLI
+      class Command < Core::CLI::Command
+
+        man_dir File.join(ROOT,'man')
+
+        bug_report_url 'https://github.com/ronin-rb/ronin-db/issues/new'
+
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/commands/add.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/command'
+require 'ronin/db/cli/uri_methods'
+require 'ronin/db/config_file'
+
+require 'command_kit/options/verbose'
+require 'command_kit/printing'
+require 'command_kit/printing/indent'
+
+module Ronin
+  module DB
+    class CLI
+      module Commands
+        #
+        # Adds an existing database to the `~/.config/ronin-db/database.yml`
+        # configuration file.
+        #
+        # ## Usage
+        #
+        #     ronin-db add [options] NAME [URI]
+        #
+        # ## Options
+        #
+        #     -A sqlite3|mysql2|postgresql|...,
+        #         --adapter                    The database adapter
+        #         --sqlite3 FILE               Alias for --adapter sqlite3 --database FILE
+        #         --mysql2                     Alias for --adapter mysql2
+        #         --postgresql                 Alias for --adapter postgresql
+        #     -H, --host HOST                  The host of the database to add
+        #     -p, --port PORT                  The port of the database to add
+        #     -u, --username USER              The user to authenticate with for the database
+        #     -P, --password PASSWORD          The password to authenticate with for the database
+        #         --read-password              Reads the database password from stdin
+        #     -D, --database NAME|PATH         The database name or path (for sqlite3)
+        #     -h, --help                       Print help information
+        #
+        # ## Arguments
+        #
+        #     NAME                             The name of the database to add.
+        #     [URI]                            The optional URI to the database.
+        #
+        class Add < Command
+
+          include URIMethods
+
+          usage '[options] NAME [URI]'
+
+          option :adapter, short: '-A',
+                           value: {
+                             type:  String,
+                             usage: 'sqlite3|mysql2|postgresql|...'
+                           },
+                           desc: 'The database adapter' do |adapter|
+                             @config[:adapter] = normalize_adapter(adapter)
+                           end
+
+          option :sqlite3, value: {
+                             type: String,
+                             usage: 'FILE'
+                           },
+                           desc: 'Alias for --adapter sqlite3 --database FILE' do |file|
+                             @config[:adapter]  = 'sqlite3'
+                             @config[:database] = normalize_sqlite3_path(file)
+                           end
+
+          option :mysql2, desc: 'Alias for --adapter mysql2' do
+            @config[:adapter] = 'mysql2'
+          end
+
+          option :postgresql, desc: 'Alias for --adapter postgresql' do
+            @config[:adapter] = 'postgresql'
+          end
+
+          option :host, short: '-H',
+                        value: {
+                          type: String,
+                          usage: 'HOST'
+                        },
+                        desc: 'The host of the database to add' do |host|
+                          @config[:host] = host
+                        end
+
+          option :port, short: '-p',
+                        value: {
+                          type: Integer,
+                          usage: 'PORT'
+                        },
+                        desc: 'The port of the database to add' do |port|
+                          @config[:port] = port
+                        end
+
+          option :username, short: '-u',
+                            value: {
+                              type: String,
+                              usage: 'USER'
+                            },
+                            desc: 'The user to authenticate with for the database' do |user|
+                              @config[:username] = user
+                            end
+
+          option :password, short: '-P',
+                            value: {
+                              type: String,
+                              usage: 'PASSWORD'
+                            },
+                            desc: 'The password to authenticate with for the database' do |password|
+                              @config[:password] = password
+                            end
+
+          option :read_password, desc: 'Reads the database password from stdin'
+
+          option :database, short: '-D',
+                            value: {
+                              type: String,
+                              usage: 'NAME|PATH'
+                            },
+                            desc: 'The database name or path (for sqlite3)' do |database|
+                              @config[:database] = database
+                            end
+
+          argument :name, desc: 'The name of the database to add'
+
+          argument :uri, required: false,
+                         desc:     'The optional URI to the database'
+
+          description 'Adds a pre-existing database to ronin-db'
+
+          man_page 'ronin-db-add.1'
+
+          # The database configuration Hash to add.
+          #
+          # @return [Hash{Symbol => Object}]
+          attr_reader :config
+
+          #
+          # Initializes the command.
+          #
+          # @param [Hash{Symbol => Object}] kwargs
+          #   Additional keyword arguments.
+          #
+          def initialize(**kwargs)
+            super(**kwargs)
+
+            @config = {}
+          end
+
+          #
+          # Runs the `ronin-db add` command.
+          #
+          def run(name,uri=nil)
+            @config = parse_uri(uri) if uri
+
+            read_password if options[:read_password]
+            validate_config!
+
+            ConfigFile.edit do |yaml|
+              yaml[name] = @config
+            end
+          end
+
+          #
+          # Reads the password from stdin and sets `config[:password]`.
+          #
+          def read_password
+            @config[:password] = ask_secret('Password')
+          end
+
+          #
+          # Validates the {#config} Hash.
+          #
+          def validate_config!
+            unless @config[:adapter]
+              print_error "must specify a URI or the --adapter option"
+              exit(1)
+            end
+
+            unless @config[:database]
+              print_error "must specify a URI or the --database option"
+              exit(1)
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/commands/asn.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/model_command'
+require 'ronin/core/cli/logging'
+
+require 'ronin/support/network/asn'
+require 'ronin/support/network/asn/list'
+
+module Ronin
+  module DB
+    class CLI
+      module Commands
+        #
+        # Queries and updates ASNs.
+        #
+        # ## Usage
+        #
+        #     ronin-db asn [options]
+        #
+        # ## Options
+        #
+        #         --db NAME                    The database to connect to (Default: default)
+        #         --db-uri URI                 The database URI to connect to
+        #     -v, --verbose                    Enables verbose output
+        #     -n, --number INT                 Searches for all ASN records with the AS number
+        #     -C XX|None|Uknown,               Searches for all ASN records with the country code
+        #         --country-code
+        #     -N, --name NAME                  Searches for all ASN records with the matching name
+        #     -I, --ip IP                      Queries the ASN record for the IP
+        #     -4, --ipv4                       Filter ASN records for only IPv4 ranges
+        #     -6, --ipv6                       Filter ASN records for only IPv6 ranges
+        #     -u, --update                     Updates the ASN records
+        #     -U, --url URI                    Overrides the default ASN list URL (Default: https://iptoasn.com/data/ip2asn-combined.tsv.gz)
+        #     -f, --file FILE                  Overrides the default ASN list file (Default: /home/postmodern/.local/share/ronin/ronin-support/ip2asn-combined.tsv.gz)
+        #     -h, --help                       Print help information
+        #
+        class Asn < ModelCommand
+
+          include Core::CLI::Logging
+
+          model_file 'ronin/db/asn'
+          model_name 'ASN'
+
+          option :number, short: '-n',
+                          value: {
+                            type: Integer,
+                          },
+                          desc: 'Searches for all ASN records with the AS number' do |number|
+                            @query_method_calls << [:with_number, [number]]
+                          end
+
+          option :country_code, short: '-C',
+                                value: {
+                                  type:  /[A-Z]{2}|None|Unknown/,
+                                  usage: 'XX|None|Uknown'
+                                },
+                                desc: 'Searches for all ASN records with the country code' do |cc|
+                                  @query_method_calls << [:with_country_code, [cc]]
+                                end
+
+          option :name, short: '-N',
+                        value: {
+                          type:  String,
+                          usage: 'NAME'
+                        },
+                        desc: 'Searches for all ASN records with the matching name' do |name|
+                          @query_method_calls << [:with_name, [name]]
+                        end
+
+          option :ip, short: '-I',
+                      value: {
+                        type:  String,
+                        usage: 'IP'
+                      },
+                      desc: 'Queries the ASN record for the IP' do |ip|
+                        @query_method_calls << [:containing_ip, [ip]]
+                      end
+
+          option :ipv4, short: '-4',
+                        desc: 'Filter ASN records for only IPv4 ranges' do
+                          @query_method_calls << [:v4]
+                        end
+
+          option :ipv6, short: '-6',
+                        desc: 'Filter ASN records for only IPv6 ranges' do
+                          @query_method_calls << [:v6]
+                        end
+
+          option :update, short: '-u',
+                          desc: 'Updates the ASN records'
+
+          option :url, short: '-U',
+                       value: {
+                         type:    String,
+                         usage:   'URI',
+                         default: Support::Network::ASN::List::URL
+                       },
+                       desc: 'Overrides the default ASN list URL'
+
+          option :file, short: '-f',
+                        value: {
+                          type:    String,
+                          usage:   'FILE',
+                          default: Support::Network::ASN::List::PATH
+                        },
+                        desc: 'Overrides the default ASN list file'
+
+          description 'Queries and updates ASNs'
+
+          man_page 'ronin-db-asn.1'
+
+          #
+          # Runs the `ronin-db asn` command.
+          #
+          def run
+            connect
+
+            if options[:update]
+              update
+            else
+              list
+            end
+          end
+
+          #
+          # Updates the ASN file and populates the database.
+          #
+          def update
+            if is_list_file_stale?
+              log_info "Updating ASN list file ..."
+              update_list_file
+            end
+
+            # clear the table instead of trying to diff the list with the table
+            model.delete_all
+
+            model.transaction do
+              parse_list_file do |record|
+                log_info "Importing #{record} ..."
+                import_asn_record(record)
+              end
+            end
+          end
+
+          #
+          # Determines if the ASN list file is stale.
+          #
+          def is_list_file_stale?
+            Support::Network::ASN::List.stale?(options[:file])
+          end
+
+          #
+          # Updates the ASN list file.
+          #
+          def update_list_file
+            Support::Network::ASN::List.update(
+              path: options[:file],
+              url:  options[:url]
+            )
+          end
+
+          #
+          # Parses the ASN list file.
+          #
+          # @yield [record]
+          #
+          # @yieldparam [Ronin::Support::Network::ASN::Record] record
+          #
+          # @return [Enumerator]
+          #
+          def parse_list_file(&block)
+            Support::Network::ASN::List.parse(options[:file],&block)
+          end
+
+          #
+          # Saves a parsed ASN record.
+          #
+          # @param [Ronin::Support::Network::ASN::Record] record
+          #
+          def import_asn_record(record)
+            asn = model.new(
+              version: if record.ipv6? then 6
+                       else                 4
+                       end,
+              range_start:  record.range.begin,
+              range_end:    record.range.end,
+              number:       record.number,
+              country_code: record.country_code,
+              name:         record.name
+            )
+
+            # bypass validations since we're importing from a known good source
+            asn.save(validate: false)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/commands/creds.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/model_command'
+require 'ronin/db/cli/modifiable'
+
+module Ronin
+  module DB
+    class CLI
+      module Commands
+        #
+        # Queries all credentials in the database.
+        #
+        # ## Usage
+        #
+        #     ronin creds [options]
+        #
+        # ## Options
+        #
+        #         --db NAME                    The database to connect to (Default: default)
+        #         --db-uri URI                 The database URI to connect to
+        #     -v, --verbose                    Enables verbose output
+        #         --add VALUE                  Adds a value to the database
+        #         --import FILE                Imports the values from the FILE into the database
+        #         --delete VALUE               Deletes a value from the database
+        #         --delete-all                 Deletes all values from the database
+        #     -u, --user [USER]                Username to search for.
+        #     -p, --password [PASS]            Password to search for.
+        #
+        class Creds < ModelCommand
+
+          include Modifiable
+
+          model_file 'ronin/db/credential'
+          model_name 'Credential'
+
+          option :user, short: '-u',
+                        value: {
+                          type:  String,
+                          usage: 'USER'
+                        },
+                        desc: 'Username to search for' do |user|
+                          @query_method_calls << [:for_user, [user]]
+                        end
+
+          option :password, short: '-P',
+                            value: {
+                              type:  String,
+                              usage: 'PASSWORD'
+                            },
+                            desc: 'Password to search for' do |password|
+                              @query_method_calls << [:with_password, [password]]
+                            end
+
+          description 'Queries all credentials in the database'
+
+          man_page 'ronin-db-creds.1'
+
+        end
+      end
+    end
+  end
+end

data/lib/ronin/db/cli/commands/edit.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+#
+# ronin-db - A common database library for managing and querying security data.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-db is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-db is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-db.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/db/cli/command'
+require 'ronin/db/config_file'
+
+require 'command_kit/edit'
+
+module Ronin
+  module DB
+    class CLI
+      module Commands
+        #
+        # Manually edits the `~/.config/ronin-db/database.yml` configuration
+        # file.
+        #
+        # ## Usage
+        #
+        #     ronin-db edit [options]
+        #
+        # ## Options
+        #
+        #     -h, --help                       Print help information
+        #
+        class Edit < Command
+
+          include CommandKit::Edit
+
+          description 'Manually edits the database.yml config'
+
+          man_page 'ronin-db-edit.1'
+
+          def run
+            edit ConfigFile::PATH
+          end
+
+        end
+      end
+    end
+  end
+end