grnsight 6.0.0 → 6.0.4

package/database/network-database/README.md

@@ -1,4 +1,4 @@
-# Network Database (Schema)
+# Network Database
 
 All files pertaining the network database live within this directory.
 
@@ -10,6 +10,16 @@ All network data is stored within the spring2022_network schema on our Postgres
 
 The schema is located within this directory at the top level in the file `schema.sql`. It defines the tables located within the spring2022_network schema. 
 
+Usage:
+To load to local database
+```
+psql postgresql://localhost/postgres -f schema.sql
+```
+To load to production database
+```
+psql <address to database> -f schema.sql
+```
+
 ### Scripts
 
 All scripts live within the subdirectory `scripts`, located in the top-level of the network database directory. 
@@ -21,9 +31,9 @@ All generated results of the scripts live in the subdirectory `script-results`,
 Within the scripts directory, there are the following files:
 
 - `generate_network.py`
-- `generate_sgd_network_from_yeastract_network.py`
 - `loader.py`
 - `filter_genes.py`
+- `generate_sgd_network_from_yeastract_network.py`
 
 #### Network Generator (and data preprocessor)
 
@@ -31,14 +41,48 @@ This script (`generate_network.py`) is a two-for-one. It first uses the yeastmin
 
 The resulting network matrices are located in `script-results/networks` and the resulting processed loader files are located within `script-results/processed-loader-files`
 
-Make sure to have all dependencies installed beforehand or you will recieve errors. (pip3 install intermine, tzlocal, etc. [see file for all imports])
+Make sure to have all dependencies installed beforehand or you will recieve errors. (pip3 install intermine, tzlocal, etc. [see file for all imports]
 
 Usage: 
 ```
 python3 generate_network.py
 ```
+#### Database Loader
+
+This script (`loader.py`) is to be used to load your preprocessed genes into the database. 
+
+This program generates direct SQL statements from the source files generated by the network generator in order to populate a relational database with those files’ data
+
+Usage: 
+To load to local database
+```
+python3 loader.py | psql postgresql://localhost/postgres
+```
+To load to production database
+```
+python3 loader.py | psql <address to database>
+```
+
+#### Filter Genes (beta functionality, not tested)
+
+This script (`filter_genes.py`) is to be used when updating a pre-existing database. It requires you to generate a new network from yeastmine using the script.`generate_network.py`. Once you generate the network, the script will access the database get all of the genes stored within. From there it will generate a csv file of all genes in the new network that are missing from your database, and all genes that have updated their display name (standard like name). You should change the database host to your localhost if you are running your own instance of postgresql and not the production database. Once finished, you can load the updated genes list using `loader.py`. *Note:* You will have to change the `GENE_SOURCE` to be the output file of the missing genes. 
+
+**Never save the password to your database in filter_genes.py! If you want, you can set up an environment variable where you store sensitive information, but for convience you will have to enter the password yourself.**
+
+Usage: 
+```
+python3 filter_genes.py
+```
+
+
 #### Generate an SGD network from a Yeastract network
 
 This script takes a network (assumed to have data from Yeastract, but it can be any given network) and gives you a network with data queried from Yeastmine (SGD). It takes the regulators and targets from a given network file, then queries Yeastmine in order to get the regulatory connections between the genes. From there, it creates a new network using the data obtained from Yeastmine. 
 
+To use, create a folder called `source-files` within the `network-database` folder. Add your network(s) to the newly created directory. Then go into the script and change the `YEASTRACT_NETWORK` to be the path of the network you would like to convert. Run the script and your SGD network (using the same genes) will be output in the `/script-results/yeastract-to-sgd-networks` directory. Change the name of the output files by editing the `SGD_MATRIX_EXCEL` and `SGD_MATRIX` variables
+
+Usage: 
+```
+python3 generate_sgd_network_from_yeastract_network.py
+```
 

package/database/network-database/schema.sql

@@ -1,6 +1,7 @@
 CREATE TABLE spring2022_network.source (
-  time_stamp TIMESTAMP,
+  time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
+  source_display_name VARCHAR,
   PRIMARY KEY(time_stamp, source)
 );
 
@@ -16,7 +17,7 @@ CREATE TABLE spring2022_network.network (
   regulator_gene_id VARCHAR,
   target_gene_id VARCHAR,
   taxon_id VARCHAR,
-  time_stamp TIMESTAMP,
+  time_stamp TIMESTAMP WITH TIME ZONE,
   source VARCHAR,
   FOREIGN KEY (regulator_gene_id, taxon_id) REFERENCES spring2022_network.gene(gene_id, taxon_id),
   FOREIGN KEY (target_gene_id, taxon_id) REFERENCES spring2022_network.gene(gene_id, taxon_id),

package/database/network-database/scripts/generate_network.py

@@ -8,8 +8,6 @@ import re
 import sys
 import os
 import datetime
-import pytz
-import tzlocal
 
 # Get Network Data from Yeastmine
 
@@ -97,7 +95,7 @@ if not os.path.exists('../script-results/processed-loader-files'):
 
 # Files to be generated
 
-# Create Networks
+# Generate Networks
 
 REGULATORS_TO_TARGETS_MATRIX = '../script-results/networks/regulators_to_targets.csv'
 REGULATORS_TO_REGULATORS_MATRIX = '../script-results/networks/regulators_to_regulators.csv'
@@ -142,25 +140,13 @@ regulator_to_regulator_file.close()
 # Source Table
 
 SOURCE_DESTINATION = '../script-results/processed-loader-files/source.csv'
-dt = datetime.datetime.now()
+timestamp = datetime.datetime.now(datetime.timezone.utc)
 
-year = dt.year
-month = f'{dt.month}'
-if len(month) == 1:
-    month = "0" + month
-day = f'{dt.day}'
-if len(day) == 1:
-    day = "0" + day
-hour = dt.hour
-minute = dt.minute
-second = dt.second
-
-
-timestamp = f'{year}-{month}-{day} {hour}:{minute}:{second}'
 source = "YeastMine - Saccharomyces Genome Database"
+display_name = "Yeastmine - SGD"
 
 source_file = open(SOURCE_DESTINATION, 'w')
-headers = f'Timestamp\tSource\n{timestamp}\t{source}'
+headers = f'Timestamp\tSource\tDisplay Name\n{timestamp}\t{source}\t{display_name}'
 source_file.write(f'{headers}\n')
 source_file.close()
 

package/database/network-database/scripts/generate_sgd_network_from_yeastract_network.py

@@ -94,7 +94,7 @@ if not os.path.exists('../script-results/yeastract-to-sgd-networks'):
 
 # Files to be generated
 
-# Create Networks
+# Generate Networks
 
 SGD_MATRIX = '../script-results/yeastract-to-sgd-networks/SGD_Regulation_matrix_profile2.csv'
 SGD_MATRIX_EXCEL = '../script-results/yeastract-to-sgd-networks/SGD_Regulation_matrix_profile2.xlsx'

package/database/network-database/scripts/loader.py

@@ -15,7 +15,7 @@ directly into a database command line utility such as `psql`.
 This function Loads Network Data Sources into the database
 """
 def LOAD_SOURCES():
-    print('COPY spring2022_network.source (time_stamp, source) FROM stdin;')
+    print('COPY spring2022_network.source (time_stamp, source, display_name) FROM stdin;')
     NETWORK_DATA_SOURCE = '../script-results/processed-loader-files/source.csv'
     with open(NETWORK_DATA_SOURCE, 'r+') as f:
         reader = csv.reader(f)
@@ -25,7 +25,8 @@ def LOAD_SOURCES():
                 r= ','.join(row).split('\t')
                 time_stamp = r[0]
                 source = r[1]
-                print(f'{time_stamp}\t{source}')
+                display_name = r[2]
+                print(f'{time_stamp}\t{source}\t{display_name}')
             row_num += 1
     print('\\.')
 
@@ -57,8 +58,8 @@ This function Loads the Network Matrix into the database
 """
 def LOAD_NETWORK():
     print('COPY spring2022_network.network (regulator_gene_id, target_gene_id, taxon_id, time_stamp, source) FROM stdin;')
-    GENE_SOURCE = '../script-results/processed-loader-files/network.csv'
-    with open(GENE_SOURCE, 'r+') as f:
+    NETWORK_SOURCE = '../script-results/processed-loader-files/network.csv'
+    with open(NETWORK_SOURCE, 'r+') as f:
         reader = csv.reader(f)
         row_num = 0
         for row in reader:

package/gdb/PG_VERSION

@@ -0,0 +1 @@
+14