grnsight 6.0.4 → 7.1.1

package/database/filter_update.py

@@ -0,0 +1,168 @@
+import os
+import csv
+from sqlalchemy import create_engine
+from sqlalchemy import text
+from constants import Constants
+from utils import Utils
+
+PROTEIN_GENE_HEADER = f'Gene ID\tDisplay Gene ID\tSpecies\tTaxon ID'
+GRN_GENE_HEADER = f'Gene ID\tDisplay Gene ID\tSpecies\tTaxon ID\tRegulator'
+
+def _get_all_data_from_database_table(database_namespace, table_name):
+    db = create_engine(os.environ['DB_URL'])
+    with db.connect() as connection:
+        result_set = connection.execute(text(f"SELECT * FROM {database_namespace}.{table_name}"))
+        return result_set.fetchall()
+    
+def _get_all_db_genes(database_namespace):
+    gene_records = _get_all_data_from_database_table(database_namespace, "gene")
+    genes = {}
+    for gene in gene_records:
+        key = (gene[0], gene[3])
+        if len(gene) > 4:
+            value = (gene[1], gene[2], gene[4])
+        else:
+            value = (gene[1], gene[2])
+        genes[key] = value
+    return genes
+
+def _get_all_db_grn_genes():
+    return _get_all_db_genes(Constants.GRN_DATABASE_NAMESPACE)
+
+def _get_all_db_ppi_genes():
+    return _get_all_db_genes(Constants.PPI_DATABASE_NAMESPACE)  
+
+def _get_all_genes():
+    db_grn_genes = _get_all_db_grn_genes()
+    db_ppi_genes = _get_all_db_ppi_genes()
+    
+    if not os.path.exists('union-gene-data'):
+        os.makedirs('union-gene-data')
+    Utils.create_union_file([Constants.PPI_GENE_SOURCE, Constants.GRN_GENE_SOURCE], Constants.GENE_DATA_DIRECTORY)
+    genes = db_grn_genes
+    
+    for gene in db_ppi_genes:
+        if gene not in genes:
+            display_gene_id, species = db_ppi_genes[gene]
+            genes[gene] = [display_gene_id, species, False]
+    
+    with open(Constants.GENE_DATA_DIRECTORY, 'r+', encoding="UTF-8") as f:
+        i = 0
+        reader = csv.reader(f)
+        for row in reader:
+            if i != 0:
+                row = row[0].split('\t')
+                gene_id = row[0]
+                display_gene_id = row[1]
+                species = row[2]
+                taxon_id = row[3]
+                regulator = row[4].capitalize()
+                key = (gene_id, taxon_id)
+                value = (display_gene_id, species, regulator)
+                if key not in genes:
+                    genes[key] = value
+                elif genes[key][0] != display_gene_id:
+                    if display_gene_id != "None":
+                        genes[key] = value
+            i+=1
+    return genes
+        
+
+def get_all_proteins():
+    protein_records = _get_all_data_from_database_table(Constants.PPI_DATABASE_NAMESPACE, "protein")
+    proteins = {}
+    for protein in protein_records:
+        key = (protein[0], protein[5])
+        value = (protein[1], protein[2], protein[3], protein[4])
+        proteins[key] = value
+    return proteins
+
+def processing_grn_gene_file():
+    return _processing_gene_file(_get_all_db_grn_genes(), is_protein=False)
+    
+def processing_ppi_gene_file():
+    return _processing_gene_file(_get_all_db_ppi_genes())
+
+def _processing_gene_file(db_genes, is_protein=True):
+    print(f'Processing gene')
+    missing_genes = {}
+    genes_to_update = {}
+    all_genes = _get_all_genes()
+    for gene in all_genes:
+        display_gene_id, species, regulator = all_genes[gene]
+        values_for_ppi = (display_gene_id, species)
+        values_for_grn = (display_gene_id, species, regulator)
+        if gene not in db_genes:
+            if is_protein:
+                missing_genes[gene] = values_for_ppi
+            else:
+                missing_genes[gene] = values_for_grn
+        elif gene in db_genes and db_genes[gene][0] != display_gene_id:
+            if db_genes[gene][0] != "None":
+                if is_protein: 
+                    genes_to_update[gene] = values_for_ppi
+                else:
+                    genes_to_update[gene] = values_for_grn  
+    return missing_genes, genes_to_update
+
+def processing_protein_file(file_path, db_proteins):
+    print(f'Processing file {file_path}')
+    ppi_missing_proteins = {}
+    ppi_proteins_to_update = {}
+    with open(file_path, 'r+', encoding="UTF-8") as f:
+        i = 0
+        reader = csv.reader(f)
+        for row in reader:
+            if i != 0:
+                row = row[0].split('\t')
+                standard_name = row[0]
+                gene_systematic_name = row[1]
+                length = float(row[2]) if row[2] != "None" else 0
+                molecular_weight = float(row[3]) if row[3] != "None" else 0
+                pi = float(row[4]) if row[4] != "None" else 0
+                taxon_id = row[5]
+                key = (standard_name, taxon_id)
+                value = (gene_systematic_name, length, molecular_weight, pi)
+                if key not in db_proteins:
+                    ppi_missing_proteins[key] = value
+                elif db_proteins[key] != value: 
+                    ppi_proteins_to_update[key] = value
+            i+=1
+    return ppi_missing_proteins, ppi_proteins_to_update
+
+def create_grn_gene_file(file_path, data):
+    _create_gene_file(file_path, GRN_GENE_HEADER, data, is_protein=False)
+    
+def create_ppi_gene_file(file_path, data):
+    _create_gene_file(file_path, PROTEIN_GENE_HEADER, data)
+
+def _create_gene_file(file_path, headers, data, is_protein=True):
+    print(f'Creating {file_path}\n')
+    gene_file = open(file_path, 'w')
+    gene_file.write(f'{headers}\n')
+    for gene in data:
+        if is_protein:
+            gene_file.write(f'{gene[0]}\t{data[gene][0]}\t{data[gene][1]}\t{gene[1]}\n')
+        else:
+            gene_file.write(f'{gene[0]}\t{data[gene][0]}\t{data[gene][1]}\t{gene[1]}\t{data[gene][2]}\n')
+    gene_file.close()
+
+def create_ppi_protein_file(file_path, data):
+    print(f'Creating {file_path}\n')
+    protein_file = open(file_path, 'w')
+    headers = f'Standard Name\tGene Systematic Name\tLength\tMolecular Weight\tPI\tTaxon ID'
+    protein_file.write(f'{headers}\n')
+    for protein in data:
+        protein_file.write(f'{protein[0]}\t{data[protein][0]}\t{data[protein][1]}\t{data[protein][2]}\t{data[protein][3]}\t{protein[1]}\n')
+    protein_file.close()
+
+# Processing gene files
+ppi_missing_genes, ppi_genes_to_update = processing_ppi_gene_file()
+grn_missing_genes, grn_genes_to_update = processing_grn_gene_file()
+ppi_missing_proteins, ppi_proteins_to_update = processing_protein_file(Constants.PPI_PROTEIN_TABLE_DATA_DIRECTORY, get_all_proteins())
+create_grn_gene_file(Constants.GRN_MISSING_GENE_DIRECTORY, grn_missing_genes)
+create_grn_gene_file(Constants.GRN_UPDATE_GENE_DIRECTORY, grn_genes_to_update)
+create_ppi_gene_file(Constants.PPI_MISSING_GENE_DIRECTORY, ppi_missing_genes)
+create_ppi_gene_file(Constants.PPI_UPDATE_GENE_DIRECTORY, ppi_genes_to_update)
+create_ppi_protein_file(Constants.PPI_MISSING_PROTEIN_DIRECTORY, ppi_missing_proteins)
+create_ppi_protein_file(Constants.PPI_UPDATE_PROTEIN_DIRECTORY, ppi_proteins_to_update)

package/database/grnsettings-database/README.md

@@ -0,0 +1,52 @@
+# GRNsettings Database
+The schema of this database lives within this directory.
+
+## The basics 
+
+### Schema
+The default database name is stored within the settings schema on our Postgres database.
+
+The schema is located within this directory at the top level of this file `schema.sql`. It creates the schema as well as defining the table located within the settings schema.
+
+1. Move to file that contains `schema.sql` file, which is under `database/grnsettings-database` folder
+
+2. Load database
+    
+    Example of loading to local database
+   
+    For Windows:
+    ```
+    psql -U postgres -f schema.sql postgresql://localhost/postgres
+    ```
+    
+    For Mac: 
+    ```
+    psql -f schema.sql postgresql://localhost/postgres
+    ```
+
+### Changing the default database name
+
+1. In order to change the default database name you would first need to login to the database using the following command:
+
+    For Windows:
+    ```
+    psql -U postgres <address to database> 
+    ```
+    For Mac:
+    ```
+    psql <address to database>
+    ```
+
+2. Then you will need to set your search path to the settings schema using the following command :
+    ```
+    SET SEARCH_PATH TO settings;
+    ```
+3. After that you will simply delete the current default database name using this command 
+    ```
+    DELETE FROM grnsettings;
+    ```
+4. And then insert the new database name using the following command
+    ```
+    INSERT INTO grnsettings(expression_dataset) VALUES ('the new default database name');
+    ```
+    _The current default database is 'dahlquist_2018'_

package/database/grnsettings-database/schema.sql

@@ -0,0 +1,4 @@
+CREATE SCHEMA settings;
+CREATE TABLE settings.grnsettings (
+  expression_dataset VARCHAR PRIMARY KEY
+);

package/database/loader.py

@@ -0,0 +1,30 @@
+import csv
+from utils import *
+from constants import Constants
+# python3 loader.py | psql postgresql://localhost/postgres
+import os
+
+if not os.path.exists('union-gene-data'):
+    os.makedirs('union-gene-data')
+    
+# Get union gene data
+Utils.create_union_file([Constants.EXPRESSION_GENE_SOURCE, Constants.PPI_GENE_SOURCE, Constants.GRN_GENE_SOURCE], Constants.GENE_DATA_DIRECTORY)
+
+# Regulatory Network
+Utils.load_sources(Constants.GRN_SOURCE_TABLE_DATA_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+Utils.load_grn_genes(Constants.GRN_GENE_SOURCE, Constants.GRN_DATABASE_NAMESPACE)
+Utils.load_grn_network(Constants.GRN_NETWORK_TABLE_DATA_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+
+# Protein-protein-interactions
+Utils.load_sources(Constants.PPI_SOURCE_TABLE_DATA_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+Utils.load_ppi_genes(Constants.PPI_GENE_SOURCE, Constants.PPI_DATABASE_NAMESPACE)
+Utils.load_proteins(Constants.PPI_PROTEIN_TABLE_DATA_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+Utils.load_ppi_network(Constants.PPI_NETWORK_TABLE_DATA_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+
+# Expression data
+Utils.load_refs(Constants.EXPRESSION_REFS_TABLE_DATA_DIRECTORY, Constants.EXPRESISON_DATABASE_NAMESPACE)
+Utils.load_expression_genes(Constants.EXPRESSION_GENE_SOURCE, Constants.EXPRESISON_DATABASE_NAMESPACE)
+Utils.load_expression_metadata(Constants.EXPRESSION_METADATA_TABLE_DATA_DIRECTORY, Constants.EXPRESISON_DATABASE_NAMESPACE)
+Utils.load_expression_data(Constants.EXPRESSION_EXPRESSION_TABLE_DATA_DIRECTORY, Constants.EXPRESISON_DATABASE_NAMESPACE)
+Utils.load_production_rates(Constants.EXPRESSION_PRODUCTION_RATE_TABLE_DATA_DIRECTORY, Constants.EXPRESISON_DATABASE_NAMESPACE)
+Utils.load_degradation_rates(Constants.EXPRESSION_DEGRADATION_RATE_TABLE_DATA_DIRECTORY, Constants.EXPRESISON_DATABASE_NAMESPACE)

package/database/loader_update.py

@@ -0,0 +1,36 @@
+import argparse
+from constants import Constants
+from utils import Utils
+
+def load_grn_data_into_database():
+    Utils.load_sources(Constants.GRN_SOURCE_TABLE_DATA_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+    Utils.update_grn_genes(Constants.GRN_UPDATE_GENE_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+    Utils.load_grn_genes(Constants.GRN_MISSING_GENE_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+    Utils.load_grn_network(Constants.GRN_NETWORK_TABLE_DATA_DIRECTORY, Constants.GRN_DATABASE_NAMESPACE)
+
+def load_ppi_data_into_database():
+    Utils.load_sources(Constants.PPI_SOURCE_TABLE_DATA_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+    Utils.update_ppi_genes(Constants.PPI_UPDATE_GENE_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+    Utils.update_ppi_proteins(Constants.PPI_UPDATE_PROTEIN_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+    Utils.load_ppi_genes(Constants.PPI_MISSING_GENE_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+    Utils.load_proteins(Constants.PPI_MISSING_PROTEIN_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+    Utils.load_ppi_network(Constants.PPI_NETWORK_TABLE_DATA_DIRECTORY, Constants.PPI_DATABASE_NAMESPACE)
+
+def main():
+    # Set up argument parsing
+    parser = argparse.ArgumentParser(description="Load data into database for GRN or PPI networks.")
+    parser.add_argument('--network', choices=['GRN', 'PPI'], required=True, help="Specify the network type to load data for")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Execute the relevant operations based on the argument
+    if args.network == 'GRN':
+        load_grn_data_into_database()
+    elif args.network == 'PPI':
+        load_ppi_data_into_database()
+    else:
+        print("Invalid network type. Please choose 'GRN' or 'PPI'.")
+
+if __name__ == "__main__":
+    main()

package/database/network-database/README.md

@@ -6,9 +6,9 @@ All files pertaining the network database live within this directory.
 
 ### Schema
 
-All network data is stored within the spring2022_network schema on our Postgres database.
+All network data is stored within the gene_regulatory_network schema on our Postgres database.
 
-The schema is located within this directory at the top level in the file `schema.sql`. It defines the tables located within the spring2022_network schema. 
+The schema is located within this directory at the top level in the file `schema.sql`. It defines the tables located within the gene_regulatory_network schema. 
 
 Usage:
 To load to local database
@@ -32,10 +32,13 @@ Within the scripts directory, there are the following files:
 
 - `generate_network.py`
 - `loader.py`
+- `generate_new_network_verion.py`
+- `loader_updates.py`
 - `filter_genes.py`
 - `generate_sgd_network_from_yeastract_network.py`
 
-#### Network Generator (and data preprocessor)
+
+#### Network Generator (and data preprocessor) (FOR FRESH DATABASE INSTALLS ONLY)
 
 This script (`generate_network.py`) is a two-for-one. It first uses the yeastmine service from the SGD database to query for all regulator genes relating to Saccharomyces cerevisiae. From there it gets all all of the targets for each regulator gene. We then construct two networks from these connections (a regulator by regulator matrix as well as a regulator by target matrix). We also construct the processed loader files, so that they are ready to load using `loader.py`.
 
@@ -47,7 +50,7 @@ Usage:
 ```
 python3 generate_network.py
 ```
-#### Database Loader
+#### Database Loader (FOR FRESH DATABASE INSTALLS ONLY)
 
 This script (`loader.py`) is to be used to load your preprocessed genes into the database. 
 
@@ -62,6 +65,35 @@ To load to production database
 ```
 python3 loader.py | psql <address to database>
 ```
+#### Network Generator (and data preprocessor) (FOR UPDATES TO EXISTING DATABASE ONLY)
+
+This script (`generate_new_network_verion.py`) is similar to its counterpart `generate_network.py`. It gets all existing genes in the database using the environment variable 'DB_URL'. You can set this environment variable on the terminal right before the command.  It uses the yeastmine service from the SGD database to query for all regulator genes relating to Saccharomyces cerevisiae. From there it gets all all of the targets for each regulator gene. We then construct two networks from these connections (a regulator by regulator matrix as well as a regulator by target matrix). We then see if the genes in the newly constructed network have any updates (i.e a gene's standard name was set or a new gene was added to the database). We also construct the processed loader files, so that they are ready to load using `loader_updates.py`.
+
+The resulting network matrices are located in `script-results/networks` and the resulting processed loader files are located within `script-results/processed-loader-files`
+
+Make sure to have all dependencies installed beforehand or you will recieve errors. (pip3 install intermine, tzlocal, etc. [see file for all imports]
+
+Usage: 
+```
+DB_URL="postgresql://[<db_user>:<password>]@<address to database>/<database name>" python3 generate_new_network_version.py
+```
+#### Database Loader (FOR UPDATES TO EXISTING DATABASE ONLY)
+
+This script (`loader_updates.py`) is to be used to load your preprocessed genes into the database. 
+
+This program generates direct SQL statements from the source files generated by the network generator in order to populate a relational database with those files’ data as well as make any needed updates to existing genes within the database. If necessary you will be prompted to enter a password.
+
+Usage: 
+To load to local database
+```
+python3 loader_updates.py | psql postgresql://localhost/postgres
+```
+To load to production database
+```
+python3 loader_updates.py | psql -h <grnsight database link> -U <user> <database name>
+
+```
+
 
 #### Filter Genes (beta functionality, not tested)
 

package/database/network-database/schema.sql

@@ -1,11 +1,11 @@
-CREATE TABLE spring2022_network.source (
+CREATE TABLE gene_regulatory_network.source (
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  source_display_name VARCHAR,
+  display_name VARCHAR,
   PRIMARY KEY(time_stamp, source)
 );
 
-CREATE TABLE spring2022_network.gene (
+CREATE TABLE gene_regulatory_network.gene (
   gene_id VARCHAR, -- systematic like name
   display_gene_id VARCHAR, -- standard like name
   species VARCHAR,
@@ -13,13 +13,13 @@ CREATE TABLE spring2022_network.gene (
   regulator BOOLEAN,
   PRIMARY KEY(gene_id, taxon_id)
 ); 
-CREATE TABLE spring2022_network.network (
+CREATE TABLE gene_regulatory_network.network (
   regulator_gene_id VARCHAR,
   target_gene_id VARCHAR,
   taxon_id VARCHAR,
   time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
-  FOREIGN KEY (regulator_gene_id, taxon_id) REFERENCES spring2022_network.gene(gene_id, taxon_id),
-  FOREIGN KEY (target_gene_id, taxon_id) REFERENCES spring2022_network.gene(gene_id, taxon_id),
-  FOREIGN KEY (time_stamp, source) REFERENCES spring2022_network.source(time_stamp, source)
+  FOREIGN KEY (regulator_gene_id, taxon_id) REFERENCES gene_regulatory_network.gene(gene_id, taxon_id),
+  FOREIGN KEY (target_gene_id, taxon_id) REFERENCES gene_regulatory_network_testing.gene(gene_id, taxon_id),
+  FOREIGN KEY (time_stamp, source) REFERENCES gene_regulatory_network_testing.source(time_stamp, source)
 ); 

package/database/network-database/scripts/generate_network.py

@@ -111,36 +111,28 @@ for regulator in regulators_to_targets:
     if regulator != None:
         regulators_list.append(regulator)
 
-print(f'Creating REGULATORS TO TARGETS MATRIX\n')
-regulator_to_target_file = open(REGULATORS_TO_TARGETS_MATRIX, 'w')
-headers = "cols regulators/rows targets"
-headers += '\t'.join(regulators_list)
-regulator_to_target_file.write(f'{headers}\n')
-for target in targets:
-  result = create_regulator_to_target_row(target, regulators_to_targets)
-  if result != False:
-    regulator_to_target_file.write(f'{result}\n')
-regulator_to_target_file.close()
-
-print(f'Creating REGULATORS TO TARGETS MATRIX\n')
-regulator_to_regulator_file = open(REGULATORS_TO_REGULATORS_MATRIX, 'w')
-headers = "cols regulators/rows targets"
-headers += '\t'.join(regulators_list)
-regulator_to_regulator_file.write(f'{headers}\n')
-for target in targets:
-  result = create_regulator_to_target_row(target, regulators_to_targets)
-  if result != False:
-    regulator_to_regulator_file.write(f'{result}\n')
-regulator_to_regulator_file.close()
-
-
+def createMatrix(fileName: str):
+    statusMessage: str = 'Creating REGULATORS TO TARGETS MATRIX\n' if fileName == REGULATORS_TO_TARGETS_MATRIX else 'Creating REGULATORS TO REGULATORS MATRIX\n'
+    print(statusMessage)
+    regulator_to_target_file = open(fileName, 'w')
+    headers = "cols regulators/rows targets\t"
+    headers += '\t'.join(regulators_list)
+    regulator_to_target_file.write(f'{headers}\n')
+    for target in targets:
+        result = create_regulator_to_target_row(target, regulators_to_targets)
+        if result:
+            regulator_to_target_file.write(f'{result}\n')
+    regulator_to_target_file.close()
+
+createMatrix(REGULATORS_TO_TARGETS_MATRIX)
+createMatrix(REGULATORS_TO_REGULATORS_MATRIX)
 
 # Create loader-files
 
 # Source Table
 
 SOURCE_DESTINATION = '../script-results/processed-loader-files/source.csv'
-timestamp = datetime.datetime.now(datetime.timezone.utc)
+timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
 
 source = "YeastMine - Saccharomyces Genome Database"
 display_name = "Yeastmine - SGD"

package/database/network-database/scripts/generate_new_network_version.py

@@ -0,0 +1,216 @@
+from __future__ import print_function
+
+from intermine.webservice import Service
+service = Service("https://yeastmine.yeastgenome.org/yeastmine/service")
+from sqlalchemy import create_engine
+
+import csv
+import re
+import sys
+import os
+import datetime
+
+
+def get_all_genes():
+    db = create_engine(os.environ['DB_URL'])
+    
+    with db.connect() as connection:
+        result_set = connection.execute(
+        f"""
+            SELECT display_gene_id, gene_id FROM gene_regulatory_network.gene
+        """)
+        result = result_set.fetchall()
+        return list(result)
+
+
+
+# Get Network Data from Yeastmine
+
+query = service.new_query("Gene")
+
+query.add_view(
+    "primaryIdentifier", "secondaryIdentifier", "symbol", "name", "sgdAlias",
+    "regulationSummary.summaryParagraph",
+    "regulationSummary.publications.pubMedId",
+    "regulationSummary.publications.citation"
+)
+query.outerjoin("regulationSummary.publications")
+
+regulators = {}
+all_genes = {}
+print("COLLECTING REGULATORS\n")
+for row in query.rows():
+    systematic_name = row["secondaryIdentifier"]
+    standard_name = row["symbol"]
+    if standard_name == None:
+        standard_name = systematic_name
+    
+    regulators[standard_name] =  systematic_name
+    all_genes[standard_name] = systematic_name
+
+regulators_to_targets = {}
+all_targets = {}
+
+
+print("COLLECTING TARGETS\n")
+for regulator in regulators:
+    query = service.new_query("Gene")
+    query.add_constraint("regulatoryRegions", "TFBindingSite")
+    query.add_view(
+        "regulatoryRegions.regulator.symbol",
+        "regulatoryRegions.regulator.secondaryIdentifier", "symbol",
+        "secondaryIdentifier", "regulatoryRegions.regEvidence.ontologyTerm.name",
+        "regulatoryRegions.regEvidence.ontologyTerm.identifier",
+        "regulatoryRegions.experimentCondition",
+        "regulatoryRegions.strainBackground",
+        "regulatoryRegions.regulationDirection",
+        "regulatoryRegions.publications.pubMedId", "regulatoryRegions.datasource",
+        "regulatoryRegions.annotationType"
+    )
+    query.add_sort_order("Gene.secondaryIdentifier", "ASC")
+    query.add_constraint("regulatoryRegions.regulator", "LOOKUP", regulator, "S. cerevisiae", code="A")
+    targets = {}
+
+    for row in query.rows():
+        target_systematic_name = row["secondaryIdentifier"]
+        target_standard_name = row["symbol"]
+        if target_standard_name == None:
+            target_standard_name = target_systematic_name
+        targets[target_standard_name] = target_systematic_name
+        all_targets[target_standard_name] = target_systematic_name
+        all_genes[target_standard_name] =  target_systematic_name
+
+    regulators_to_targets[regulator] = { "systematic_name": regulators[regulator], "targets": targets}
+
+
+
+def create_regulator_to_target_row(target, all_regulators):
+    result = "" + target
+    for regulator in all_regulators:
+        if target in all_regulators[regulator]["targets"]:
+            result += "\t" + "1"
+        else: 
+            result += "\t" + "0"
+    return result
+
+
+# Create files
+
+# Create folder paths 
+if not os.path.exists('../script-results'):
+    os.makedirs('../script-results')
+
+if not os.path.exists('../script-results/networks'):
+    os.makedirs('../script-results/networks')
+
+if not os.path.exists('../script-results/processed-loader-files'):
+    os.makedirs('../script-results/processed-loader-files')
+
+
+
+# Files to be generated
+
+# Generate Networks
+
+REGULATORS_TO_TARGETS_MATRIX = '../script-results/networks/regulators_to_targets.csv'
+REGULATORS_TO_REGULATORS_MATRIX = '../script-results/networks/regulators_to_regulators.csv'
+
+
+targets = []
+for target in all_targets:
+    if target != None:
+        targets.append(target)
+
+regulators_list = []
+for regulator in regulators_to_targets:
+    if regulator != None:
+        regulators_list.append(regulator)
+
+# this is the same method used in generate_network.py to create Regulators to Targets Matrix and Regulators to Regulators Matrix
+def createMatrix(fileName: str):
+    statusMessage: str = 'Creating REGULATORS TO TARGETS MATRIX\n' if fileName == REGULATORS_TO_TARGETS_MATRIX else 'Creating REGULATORS TO REGULATORS MATRIX\n'
+    print(statusMessage)
+    regulator_to_target_file = open(fileName, 'w')
+    headers = "cols regulators/rows targets\t"
+    headers += '\t'.join(regulators_list)
+    regulator_to_target_file.write(f'{headers}\n')
+    for target in targets:
+        result = create_regulator_to_target_row(target, regulators_to_targets)
+        if result:
+            regulator_to_target_file.write(f'{result}\n')
+    regulator_to_target_file.close()
+
+createMatrix(REGULATORS_TO_TARGETS_MATRIX)
+createMatrix(REGULATORS_TO_REGULATORS_MATRIX)
+
+# Create loader-files
+
+# Source Table
+
+SOURCE_DESTINATION = '../script-results/processed-loader-files/source.csv'
+timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
+
+source = "YeastMine - Saccharomyces Genome Database"
+display_name = "Yeastmine - SGD"
+
+source_file = open(SOURCE_DESTINATION, 'w')
+headers = f'Timestamp\tSource\tDisplay Name\n{timestamp}\t{source}\t{display_name}'
+source_file.write(f'{headers}\n')
+source_file.close()
+
+# Gene Table
+
+GENE_DESTINATION = '../script-results/processed-loader-files/gene.csv'
+
+database_genes = {g[1] : g[0] for g in get_all_genes()}
+database_standard_names = [g[0] for g in get_all_genes()]
+database_systematic_names = [g[1] for g in get_all_genes()]
+species = "Saccharomyces cerevisiae"
+taxon_id = "559292"
+genes_to_update = {}
+print(f'Creating gene.csv\n')
+gene_file = open(GENE_DESTINATION, 'w')
+headers = f'Gene ID\tDisplay Gene ID\tSpecies\tTaxon ID\tRegulator'
+gene_file.write(f'{headers}\n')
+for gene in all_genes:
+    if gene not in database_standard_names:
+        if all_genes[gene] not in database_systematic_names:
+            if gene in regulators:
+                gene_file.write(f'{all_genes[gene]}\t{gene}\t{species}\t{taxon_id}\ttrue\n')
+            else:
+                gene_file.write(f'{all_genes[gene]}\t{gene}\t{species}\t{taxon_id}\tfalse\n')
+        else:
+            # if systematic name is found, then we need to update the gene
+            genes_to_update[gene] = all_genes[gene]
+gene_file.close()
+
+# Gene Table Updates
+
+GENE_UPDATES_DESTINATION = '../script-results/processed-loader-files/gene_update.csv'
+
+
+print(f'Creating gene_update.csv\n')
+gene_file = open(GENE_UPDATES_DESTINATION, 'w')
+headers = f'Gene ID\tDisplay Gene ID\tRegulator'
+gene_file.write(f'{headers}\n')
+for gene in genes_to_update:
+    if gene in regulators:
+        gene_file.write(f'{all_genes[gene]}\t{gene}\ttrue\n')
+    else:
+        gene_file.write(f'{all_genes[gene]}\t{gene}\tfalse\n')
+gene_file.close()
+
+
+# Network Table
+
+NETWORK_DESTINATION = '../script-results/processed-loader-files/network.csv'
+
+
+print(f'Creating network.csv\n')
+network_file = open(NETWORK_DESTINATION, 'w')
+headers = f'Regulator Gene ID\tTarget Gene ID\tTaxon ID\tTimestamp\tSource'
+network_file.write(f'{headers}\n')
+for gene in regulators_to_targets:
+    for target_gene in regulators_to_targets[gene]["targets"]:
+        network_file.write(f'{regulators_to_targets[gene]["systematic_name"]}\t{regulators_to_targets[gene]["targets"][target_gene]}\t{taxon_id}\t{timestamp}\t{source}\n')
+network_file.close()