grnsight 6.0.4 → 7.1.1

package/database/protein-protein-database/README.md

@@ -0,0 +1,71 @@
+# Protein-Protein Database 
+
+All files pertaining the protein-protein database live within this directory.
+
+## The basics
+
+#### Schema
+
+All network data is stored within the protein_protein_interactions schema on our Postgres database.
+
+The schema is located within this directory at the top level in the file `schema.sql`. It defines the tables located within the protein_protein_interactions schema. 
+
+Usage:
+To load to local database
+```
+psql -f schema.sql postgresql://localhost/postgres
+```
+To load to production database 
+```
+psql -f schema.sql <address to database>
+```
+
+### Scripts
+
+All scripts live within the subdirectory `scripts`, located in the top-level of the network database directory. 
+
+Any source files required to run the scripts live within the subdirectory `source-files`, located in the top-level of the network database directory. As source files may be large, you must create this directory yourself and add any source files you need to use there. 
+
+All generated results of the scripts live in the subdirectory `script-results`, located in the top-level of the network database directory. Currently, all scripts that generate code create the directory if it does not currently exist. When adding a new script that generates resulting code, best practice is to create the script-results directory and any subdirectories if it does not exist, in order to prevent errors and snafus for recently cloned repositories.
+
+Within the scripts directory, there are the following files:
+
+- `generate_protein_network.py`
+- `remove_duplicates.sh`
+- `loader.py`
+
+#### Data Generator
+
+This script (`generate_protein_network.py`) generates the genes, protein information and the physical interactions between these genes from Yeastmine; then it writes this into the csv files used to load the database. Please make sure you have enough time (around 1.5 - 2 hours) to run this script. The files (`gene.csv`), (`physical_interaction.csv`), (`protein.csv`) will be generated in the script-results directory located in the sub-directory processed-loader-files.
+
+    Usage: 
+    ```
+    python3 generate_protein_network.py
+    ```
+
+Once you have finished generating the loader files, you need to remove duplicate entries from the physical interactions file. The bash script (`remove_duplicates.sh`) does this for you. The resultant file (`no_dupe.csv`)will be generated in the script-results directory located in the sub-directory processed-loader-files. If your machine doesn't support bash shell scripts, then you have to make a new script that removes duplicate lines from a file and writes the results to a file. Sorry!
+
+    Usage: 
+    ```
+    chmod u+x remove_duplicates.sh
+    
+    ./remove_duplicates.sh
+    ```
+
+#### Database Loader
+
+This script (`loader.py`) is to be used to load your preprocessed expression data into the database. 
+
+This program generates direct SQL statements from the source files generated by the data preprocessor in order to populate a relational database with those files’ data
+
+Note: You may get an error saying that there was a duplicate protein. You have to manually check which protein was being inserted twice, go to the SGD website (or Yeastmine) and confirm the correct protein gene interaction. Currently this occurs with the protein 'Aad6p'. To fix it go to your protein.csv file and make sure that 'Aad6p' is paired with the gene 'YFL056C', and 'Aad16p' is paired with the gene 'YFL057C'. If any other issues arise, you must manually confirm on the SGD website. Sorry!
+
+Usage: 
+To load to local database
+```
+python3 loader.py | psql postgresql://localhost/postgres
+```
+To load to production database
+```
+python3 loader.py | psql <path to database>
+```

package/database/protein-protein-database/schema.sql

@@ -0,0 +1,37 @@
+CREATE TABLE protein_protein_interactions.source (
+  time_stamp TIMESTAMP WITH TIME ZONE,
+  source VARCHAR,
+  display_name VARCHAR,
+  PRIMARY KEY(time_stamp, source)
+);
+
+CREATE TABLE protein_protein_interactions.gene (
+  gene_id VARCHAR, -- systematic like name
+  display_gene_id VARCHAR, -- standard like name
+  species VARCHAR,
+  taxon_id VARCHAR,
+  PRIMARY KEY(gene_id, taxon_id)
+); 
+
+CREATE TABLE protein_protein_interactions.protein (
+  standard_name VARCHAR PRIMARY KEY,
+  gene_systematic_name VARCHAR,
+  length FLOAT,
+  molecular_weight FLOAT,
+  PI FLOAT,
+  taxon_id VARCHAR,
+  FOREIGN KEY (gene_systematic_name, taxon_id) REFERENCES protein_protein_interactions_testing.gene(gene_id, taxon_id)
+);
+
+CREATE TABLE protein_protein_interactions.physical_interactions (
+  protein1 VARCHAR,
+  protein2 VARCHAR,
+  interaction_detection_methods_identifier VARCHAR,
+  experiment_name VARCHAR,
+  time_stamp TIMESTAMP WITH TIME ZONE,
+  source VARCHAR,
+  FOREIGN KEY (protein1) REFERENCES protein_protein_interactions_testing.protein(standard_name),
+  FOREIGN KEY (protein2) REFERENCES protein_protein_interactions_testing.protein(standard_name),
+  FOREIGN KEY (time_stamp, source) REFERENCES protein_protein_interactions_testing.source(time_stamp, source),
+  CONSTRAINT unique_physical_interaction UNIQUE (protein1, protein2, interaction_detection_methods_identifier, experiment_name, time_stamp, source)
+);

package/database/protein-protein-database/scripts/generate_protein_network.py

@@ -0,0 +1,227 @@
+from __future__ import print_function
+
+from intermine.webservice import Service
+service = Service("https://yeastmine.yeastgenome.org/yeastmine/service")
+
+import csv
+import re
+import sys
+import os
+import gc
+import datetime
+
+
+# Create files
+
+# Create folder paths 
+if not os.path.exists('../script-results'):
+    os.makedirs('../script-results')
+
+if not os.path.exists('../script-results/processed-loader-files'):
+    os.makedirs('../script-results/processed-loader-files')
+
+
+
+# Files to be generated
+GENE_FILE = '../script-results/processed-loader-files/gene.csv'
+PROTEIN_FILE = '../script-results/processed-loader-files/protein.csv'
+PHYSICAL_INTERACTION_FILE = '../script-results/processed-loader-files/physical_interaction.csv'
+SOURCE_DESTINATION = '../script-results/processed-loader-files/source.csv'
+
+# Instantiate Source variables
+timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
+source = "YeastMine - Saccharomyces Genome Database"
+display_name = "Yeastmine - SGD"
+
+# Get Network Data from Yeastmine
+
+def getPhysicalInteractions(gene):
+    print("Query data from Yeastmine to get Physical Interactions")
+    query = service.new_query("Gene")
+    query.add_constraint("interactions.participant2", "Gene")
+    query.add_view(
+        "primaryIdentifier", "symbol", "secondaryIdentifier", "sgdAlias", "name",
+        "organism.shortName", "interactions.details.annotationType",
+        "interactions.details.role1", "interactions.participant2.symbol",
+        "interactions.participant2.secondaryIdentifier",
+        "interactions.details.experiment.interactionDetectionMethods.identifier",
+        "interactions.details.experiment.name",
+        "interactions.details.relationshipType", "interactions.details.note"
+    )
+    query.add_constraint("organism.shortName", "=", "S. cerevisiae", code="B")
+    query.add_constraint("interactions.details.relationshipType", "=", "physical", code="C")
+    query.add_constraint("Gene", "LOOKUP", gene, code="A")
+    return query
+
+def getProteinFromGene(gene):
+    print("Query data from Yeastmine to get Protein information")
+    query = service.new_query("Gene")
+    query.add_view(
+        "primaryIdentifier", "proteins.symbol", "sgdAlias", "proteins.length",
+        "proteins.molecularWeight", "proteins.pI", "featureType", "qualifier",
+        "description", "proteins.sequence.residues"
+    )
+    query.add_constraint("organism.shortName", "=", "S. cerevisiae", code="B")
+    query.add_constraint("Gene", "LOOKUP", gene, code="A")
+    return query
+
+def getAllProteins():
+    print("Query data from Yeastmine to get all proteins")
+
+    query = service.new_query("Protein")
+
+    query.add_view(
+        "genes.primaryIdentifier", "genes.secondaryIdentifier", "symbol", "length",
+        "molecularWeight", "pI", "genes.featureType", "genes.qualifier",
+        "genes.sgdAlias", "genes.description"
+    )
+    query.add_sort_order("Protein.symbol", "ASC")
+    query.add_constraint("genes.featureType", "=", "transposable_element_gene", code="G")
+    query.add_constraint("genes.featureType", "=", "ORF", code="F")
+    query.add_constraint("genes.status", "=", "Active", code="D")
+    query.add_constraint("genes.featureType", "=", "blocked_reading_frame", code="E")
+    query.add_constraint("genes.featureType", "=", "intein_encoding_region", code="H")
+    query.add_constraint("organism.name", "=", "Saccharomyces cerevisiae", code="A")
+    query.set_logic("A and D and (F or G or E or H)")
+
+    return query
+
+
+
+query = getAllProteins()
+all_proteins = {}
+
+genes = {
+    #  stored as gene sysyematic name : {
+        #  proteins : {protein standard name : {protein info}}
+    # }
+}
+print("COLLECTING PROTEINS FROM QUERY RESULTS\n")
+
+for row in query.rows():
+    gene_systematic_name = row["genes.secondaryIdentifier"]
+    protein_standard_name = row["symbol"]
+    length =  row["length"]
+    molecular_weight = row["molecularWeight"]
+    PI = row["pI"]
+    genes[gene_systematic_name] = {
+        "standard_name" : None,
+        "protein" : {
+            "standard_name": protein_standard_name,
+                "length": length, 
+                "molecular_weight": molecular_weight,
+                "PI": PI
+        }      
+    }
+
+        
+print("COLLECTING/WRITING INTERACTIONS\n")
+file = open(PHYSICAL_INTERACTION_FILE,"w")
+print(f"Open file {PHYSICAL_INTERACTION_FILE} and write data into that file")
+file.write(f"Protein1\tProtein2\tInteraction Detection Methods Identifier\tExperiment Name\tTime_Stamp\tSource\n")
+
+
+exceptions = []
+print("Processing Physical Interactions")
+for gene in genes:
+    query = getPhysicalInteractions(gene)
+    first_row = True
+
+    for row in query.rows():
+        gene1 = row["secondaryIdentifier"]
+        gene2 = row["interactions.participant2.secondaryIdentifier"]
+        if first_row:
+            # update the gene's standard name
+            genes[gene]["standard_name"] = row["symbol"] if row["symbol"] != None else gene
+            first_row = False
+        if gene2 in genes:
+            g = sorted([genes[gene1]["protein"]["standard_name"], genes[gene2]["protein"]["standard_name"]])
+            idmi = row["interactions.details.experiment.interactionDetectionMethods.identifier"]
+            exp_name = row["interactions.details.experiment.name"]
+            
+            if gene2 in genes and gene1 in genes:
+                file.write(f'{g[0]}\t{g[1]}\t{idmi}\t{exp_name}\t{timestamp}\t{source}\n')
+        else: 
+            exceptions.append(gene2)
+
+print("Handling Exceptions")
+failed_genes = []
+while exceptions != None:
+    acceptable_genes = []
+    for gene in exceptions:
+        query = getProteinFromGene(gene)
+        rows = query.rows()
+        for row in rows:
+            acceptable_genes.append(gene)
+            protein_standard_name = row["proteins.symbol"]
+            length =  row["proteins.length"]
+            molecular_weight = row["proteins.molecularWeight"]
+            PI = row["proteins.pI"]
+            genes[gene] = {
+                "standard_name" : None,
+                "protein" : {
+                    "standard_name": protein_standard_name,
+                        "length": length, 
+                        "molecular_weight": molecular_weight,
+                        "PI": PI
+            }      
+        }
+        if len(rows) == 0:
+            failed_genes.append(gene)  
+            
+    more_exceptions = []
+    for gene in acceptable_genes:
+        query = getPhysicalInteractions(gene)
+        first_row = True
+        for row in query.rows():
+            gene1 = row["secondaryIdentifier"]
+            gene2 = row["interactions.participant2.secondaryIdentifier"]
+            if first_row:
+                # update the gene's standard name
+                genes[gene]["standard_name"] = row["symbol"] if row["symbol"] != None else gene
+                first_row = False
+            if gene2 in genes:
+                g = sorted([genes[gene1]["protein"]["standard_name"], genes[gene2]["protein"]["standard_name"]])
+                idmi = row["interactions.details.experiment.interactionDetectionMethods.identifier"]
+                exp_name = row["interactions.details.experiment.name"]
+                
+                if gene2 in genes and gene1 in genes:
+                    file.write(f'{g[0]}\t{g[1]}\t{idmi}\t{exp_name}\t{timestamp}\t{source}\n')
+            elif gene not in failed_genes: 
+                more_exceptions.append(gene2)
+    if len(more_exceptions) == 0:
+        exceptions = None
+    else :
+        exceptions = more_exceptions
+    
+    
+file.close()
+
+# Source Table
+
+print(f"Completed {PHYSICAL_INTERACTION_FILE} Starting{SOURCE_DESTINATION}")
+
+source_file = open(SOURCE_DESTINATION, 'w')
+headers = f'Timestamp\tSource\tDisplay Name\n{timestamp}\t{source}\t{display_name}'
+source_file.write(f'{headers}\n')
+source_file.close()
+
+
+species = "Saccharomyces cerevisiae"
+taxon_id = "559292"
+
+# create gene csv
+print(f"Completed {SOURCE_DESTINATION} Starting{GENE_FILE}")
+file = open(GENE_FILE,"w")
+file.write(f"Gene ID\tDisplay Gene ID\tSpecies\tTaxon ID\n")
+for gene in genes:
+    file.write(f"{gene}\t{genes[gene]['standard_name']}\t{species}\t{taxon_id}\n")
+file.close()
+
+# create protein csv
+print(f"Completed {GENE_FILE} Starting{PROTEIN_FILE}")
+file = open(PROTEIN_FILE, "w")
+file.write(f"Standard Name\tGene Systematic Name\tLength\tMolecular Weight\tPI\tTaxon ID\n")
+for gene in genes:
+    file.write(f"{genes[gene]['protein']['standard_name']}\t{gene}\t{genes[gene]['protein']['length']}\t{genes[gene]['protein']['molecular_weight']}\t{genes[gene]['protein']['PI']}\t{taxon_id}\n")
+file.close()

package/database/protein-protein-database/scripts/remove_duplicates.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+output="../script-results/processed-loader-files/physical_interaction_no_dupe.csv"
+source="../script-results/processed-loader-files/physical_interaction.csv"
+awk '!x[$0]++' $source > $output