@teselagen/bio-parsers 0.1.27 → 0.1.28

package/src/snapgeneToJson.js

@@ -0,0 +1,245 @@
+//note: Huge credit and thanks go to IsaacLuo from whose python repository this code was adapted
+// https://github.com/IsaacLuo/SnapGeneFileReader
+
+import bufferpack from "bufferpack";
+import { StringDecoder } from "string_decoder";
+import buffer from "buffer";
+
+import getArrayBufferFromFile from "./utils/getArrayBufferFromFile";
+import createInitialSequence from "./utils/createInitialSequence";
+import validateSequenceArray from "./utils/validateSequenceArray";
+import flattenSequenceArray from "./utils/flattenSequenceArray";
+import { get } from "lodash";
+import { XMLParser } from "fast-xml-parser";
+import extractFileExtension from "./utils/extractFileExtension";
+
+const Buffer = buffer.Buffer
+
+async function snapgeneToJson(fileObj, options = {}) {
+  try {
+    const returnVal = createInitialSequence(options);
+    const arrayBuffer = await getArrayBufferFromFile(fileObj);
+    const ext = extractFileExtension(options.fileName);
+    let isProtein = options.isProtein;
+    if (ext && /^(prot)$/.test(ext)) {
+      isProtein = true;
+      options.isProtein = true;
+    }
+    let offset = 0;
+    // eslint-disable-next-line no-inner-declarations
+    function read(size, fmt) {
+      const buffer = Buffer.from(arrayBuffer.slice(offset, size + offset));
+      offset += size;
+      if (fmt) {
+        const decoder = new StringDecoder(fmt);
+        const toRet = decoder.write(buffer);
+        return toRet;
+      } else {
+        return buffer;
+      }
+    }
+    // eslint-disable-next-line no-inner-declarations
+    async function unpack(size, mode) {
+      const buffer = await read(size);
+      const unpacked = await bufferpack.unpack(">" + mode, buffer);
+      if (unpacked === undefined) return undefined;
+      return await unpacked[0];
+    }
+
+    await read(1); //read the first byte
+    // READ THE DOCUMENT PROPERTIES
+    const length = await unpack(4, "I");
+    const title = await read(8, "ascii");
+    if (length !== 14 || title !== "SnapGene") {
+      throw new Error("Wrong format for a SnapGene file !");
+    }
+    const data = await {
+      ...returnVal.parsedSequence,
+      isProtein,
+      isDNA: !!(await unpack(2, "H")) && !isProtein,
+      exportVersion: await unpack(2, "H"),
+      importVersion: await unpack(2, "H"),
+      features: [],
+    };
+    while (offset <= arrayBuffer.byteLength) {
+      // # READ THE WHOLE FILE, BLOCK BY BLOCK, UNTIL THE END
+      const next_byte = await read(1);
+      // # next_byte table
+      // # 0: dna sequence
+      // # 1: compressed DNA
+      // # 2: unknown
+      // # 3: unknown
+      // # 5: primers
+      // # 6: notes
+      // # 7: history tree
+      // # 8: additional sequence properties segment
+      // # 9: file Description
+      // # 10: features
+      // # 11: history node
+      // # 13: unknown
+      // # 16: alignable sequence
+      // # 17: alignable sequence
+      // # 18: sequence trace
+      // # 19: Uracil Positions
+      // # 20: custom DNA colors
+
+      const block_size = await unpack(4, "I");
+      if (ord(next_byte) === 21 || ord(next_byte) === 0) {
+        //   # READ THE SEQUENCE AND ITS PROPERTIES
+        const props = await unpack(1, "b");
+        const binaryRep = dec2bin(props);
+
+        data.circular = isFirstBitA1(binaryRep);
+        const size = block_size - 1;
+        if (size < 0) return;
+        data.size = isProtein ? size * 3 : size;
+        //   data["dna"] = {
+        //     topology="circular" if props & 0x01 else "linear",
+        //     strandedness="double" if props & 0x02 > 0 else "single",
+        //     damMethylated=props & 0x04 > 0,
+        //     dcmMethylated=props & 0x08 > 0,
+        //     ecoKIMethylated=props & 0x10 > 0,
+        //     length=block_size - 1
+        //   }
+        data.sequence = await read(size, "utf8");
+      } else if (ord(next_byte) === 10) {
+        //   # READ THE FEATURES
+        const strand_dict = {
+          // [strand, arrowheadType]
+          0: [1, "NONE"], // non-directional feature (in that case, the attribute is generally absent altogether)
+          1: [1, "TOP"], // forward strand
+          2: [-1, "BOTTOM"], // reverse strand
+          3: [1, "BOTH"], // bi-directional feature
+        };
+        const xml = await read(block_size, "utf8");
+        const b = new XMLParser({
+          ignoreAttributes: false,
+          attributeNamePrefix: "",
+          isArray: (name) => name === "Feature" || name === "Segment",
+        }).parse(xml);
+        const { Features: { Feature = [] } = {} } = b;
+        data.features = [];
+        Feature.forEach((feat) => {
+          const { directionality, Segment = [], name, type } = feat;
+          // let color;
+          let maxStart = 0;
+          let maxEnd = 0;
+          const locations =
+            Segment &&
+            Segment.map((seg) => {
+              if (!seg) throw new Error("invalid feature definition");
+              const { range } = seg;
+              // color = seg.color;
+              let { start, end } = getStartAndEndFromRangeString(range);
+              start = isProtein ? start * 3 : start;
+              end = isProtein ? end * 3 + 2 : end;
+              maxStart = Math.max(maxStart, start);
+              maxEnd = Math.max(maxEnd, end);
+              return {
+                start,
+                end,
+              };
+            });
+
+          data.features.push({
+            name,
+            type,
+            ...(locations?.length > 1 && { locations }),
+            strand: directionality ? strand_dict[directionality][0] : 1,
+            arrowheadType: directionality ? strand_dict[directionality][1] : "NONE",
+            start: maxStart,
+            end: maxEnd,
+            // color,
+          });
+        });
+      } else if (ord(next_byte) === 6) {
+        //       # READ THE NOTES
+
+        const xml = await read(block_size, "utf8");
+        const b = new XMLParser({}).parse(xml);
+        const name = get(b, "Notes.CustomMapLabel");
+        if (name) {
+          data.name = name;
+        }
+
+        const description = get(b, "Notes.Description");
+        if (description && typeof description === "string") {
+          data.description = description
+            .replace("<html><body>", "")
+            .replace("</body></html>", ""); //fixes https://github.com/TeselaGen/ve-sequence-parsers/issues/225
+        }
+      } else {
+        // # WE IGNORE THE WHOLE BLOCK
+        await read(block_size); //we don't do anything with this
+      }
+    }
+    returnVal.parsedSequence = data;
+    return validateSequenceArray(
+      flattenSequenceArray([returnVal], options),
+      options
+    );
+  } catch (e) {
+    console.error("Error trying to parse file as snapgene:", e);
+    return [
+      {
+        success: false,
+        messages: ["Import Error: Invalid File"],
+      },
+    ];
+  }
+}
+
+function getStartAndEndFromRangeString(rangestring) {
+  const [start, end] = rangestring.split("-");
+  return {
+    start: start - 1,
+    end: end - 1,
+  };
+}
+
+function ord(string) {
+  //  discuss at: http://locutus.io/php/ord/
+  // original by: Kevin van Zonneveld (http://kvz.io)
+  // bugfixed by: Onno Marsman (https://twitter.com/onnomarsman)
+  // improved by: Brett Zamir (http://brett-zamir.me)
+  //    input by: incidence
+  //   example 1: ord('K')
+  //   returns 1: 75
+  //   example 2: ord('\uD800\uDC00'); // surrogate pair to create a single Unicode character
+  //   returns 2: 65536
+  const str = string + "";
+  const code = str.charCodeAt(0);
+  if (code >= 0xd800 && code <= 0xdbff) {
+    // High surrogate (could change last hex to 0xDB7F to treat
+    // high private surrogates as single characters)
+    const hi = code;
+    if (str.length === 1) {
+      // This is just a high surrogate with no following low surrogate,
+      // so we return its value;
+      return code;
+      // we could also throw an error as it is not a complete character,
+      // but someone may want to know
+    }
+    const low = str.charCodeAt(1);
+    return (hi - 0xd800) * 0x400 + (low - 0xdc00) + 0x10000;
+  }
+  if (code >= 0xdc00 && code <= 0xdfff) {
+    // Low surrogate
+    // This is just a low surrogate with no preceding high surrogate,
+    // so we return its value;
+    return code;
+    // we could also throw an error as it is not a complete character,
+    // but someone may want to know
+  }
+  return code;
+}
+
+export default snapgeneToJson;
+
+function dec2bin(dec) {
+  return (dec >>> 0).toString(2);
+}
+
+function isFirstBitA1(num) {
+  return Number(num.toString().split("").pop()) === 1;
+}

package/src/utils/NameUtils.js

@@ -0,0 +1,10 @@
+// Basically a copy of 'Teselagen.utils.NameUtils' for use within workers.
+
+/**
+ * Reformat name to replaces whitespace with underscores.
+ * @param {string} pName
+ * @returns {string} New name.
+ */
+export const reformatName = function (pName) {
+  return pName.toString().replace(/ /g, '_');
+};

package/src/utils/ParserUtil.js

@@ -0,0 +1,93 @@
+const ParserUtil = {};
+ParserUtil.postProcessGenbankFeature = function(feat) {
+	let name = null;
+	// let nameIndex = null;
+
+	// let hasName = false;
+	let usingLabel = false;
+	let usingGene = false;
+
+	for (let j = 0; j < feat.notes.length; j++) {
+		const note = feat.notes[j];
+		const key = note.name;
+		const value = note.value;
+
+		// SET THE LABEL FIELD. DO NOT STORE AS AN ATTRIBUTE
+
+		if (this.isAGenbankFeatureLabel(key)) {
+			// Priority for name attributes is: 'label' > 'gene' > 'organism'.
+			// We check to see if the current name is from a lower-priority
+			// attribute. If it is, we store it as an attribute and then
+			// replace it with the current higher-priority attribute.
+
+			if (key === "label") {
+				// Label has top priority.
+
+				name = value;
+				// nameIndex = j;
+
+				usingLabel = true;
+			}
+			else if (key === "gene") {
+
+				// If we're not using the label for the name, use the
+				// current 'gene' attribute. If we are using label for
+				// the name, just save the current attribute as a normal
+				// attribute.
+				if (!usingLabel) {
+
+					name = value;
+					// nameIndex = j;
+
+					usingGene = true;
+				}
+			}
+			else if (!usingLabel && !usingGene) {
+				// If we don't have a label from either a 'gene' or a
+				// 'label' field, use the current field as the name.
+
+				name = value;
+				// nameIndex = j;
+
+			}
+
+			// hasName = true;
+		}
+	}
+
+	feat.name = name || "";
+	// if(nameIndex !== null) {
+	// 	feat.notes.splice(nameIndex, 1);
+	// }
+	//
+	// if(feat.locations.length > 0) {
+	// 	var loc = feat.locations[0];
+	// 	feat.start = loc.start;
+	// 	feat.end = loc.end;
+	// }
+	// else {
+	// 	feat.start = null;
+	// 	feat.end = null;
+	// }
+
+	return feat;
+};
+
+
+
+/**
+ * isAFeatureLabel
+ * @param {string} name Name of a attribute or qualifier
+ * @return {boolean} isALabel
+ */
+ParserUtil.isAGenbankFeatureLabel = function(name) {
+	if (name === "label" || name === "name" || name === "ApEinfo_label" ||
+		name === "note" || name === "gene" || name === "organism" || name === "locus_tag") {
+
+		return true;
+	}
+	else {
+		return false;
+	}
+};
+export default ParserUtil;

package/src/utils/cleanUpTeselagenJsonForExport.js

@@ -0,0 +1,13 @@
+import { cloneDeep, forEach } from "lodash";
+
+export default function cleanUpTeselagenJsonForExport(tgJson) {
+	const seqData = cloneDeep(tgJson);
+	if (!seqData) return seqData
+	delete seqData.cutsites;
+	delete seqData.orfs;
+	forEach(seqData.translations,(t)=>{
+		delete t.aminoAcids
+	})
+	return seqData
+}
+

package/src/utils/constants.js

@@ -0,0 +1,24 @@
+export const untitledSequenceName = 'Untitled Sequence';
+
+export const gbDivisions = {
+    // https://www.ncbi.nlm.nih.gov/Sitemap/samplerecord.html#GenBankDivisionB
+    PRI: true,  //- primate sequences
+    ROD: true,  //- rodent sequences
+    MAM: true,  //- other mammalian sequences
+    VRT: true,  //- other vertebrate sequences
+    INV: true,  //- invertebrate sequences
+    PLN: true,  //- plant, fungal, and algal sequences
+    BCT: true,  //- bacterial sequences
+    VRL: true,  //- viral sequences
+    PHG: true,  //- bacteriophage sequences
+    SYN: true,  //- synthetic sequences
+    UNA: true,  //- unannotated sequences
+    EST: true,  //- EST sequences (expressed sequence tags)
+    PAT: true,  //- patent sequences
+    STS: true,  //- STS sequences (sequence tagged sites)
+    GSS: true,  //- GSS sequences (genome survey sequences)
+    HTG: true,  //- HTG sequences (high-throughput genomic sequences)
+    HTC: true,  //- unfinished high-throughput cDNA sequencing
+    ENV: true,  //- environmental sampling sequences
+    CON: true,  //- sequence assembly instructions on how to construct contigs from multiple GenBank records.
+  }

package/src/utils/convertOldSequenceDataToNewDataType.js

@@ -0,0 +1,64 @@
+import { isRangeWithinRange } from '@teselagen/range-utils';
+
+export default function convertOldSequenceDataToNewDataType(
+  oldTeselagenJson,
+  opts
+) {
+  if (opts && opts.splitLocations) {
+    //after the file has been parsed, but before it's been saved, check for features with multiple locations and split them
+    oldTeselagenJson &&
+      oldTeselagenJson.features.forEach(function(feature) {
+        if (feature.locations && feature.locations[0]) {
+          if (feature.locations.length > 1) {
+            for (let i = 1; i < feature.locations.length; i++) {
+              //start at index 1, not 0!
+              //for every location except for the first one,
+              const location = feature.locations[i];
+              const clonedFeature = JSON.parse(JSON.stringify(feature));
+              clonedFeature.start = location.start;
+              clonedFeature.end = location.end;
+              delete clonedFeature.locations; //This array is no longer used to get start and end bp and doesn't need to be in db
+              //clonedFeature.locations = []; //strip the locations from the cloned feature (we won't be using locations whatsoever in our app)
+              oldTeselagenJson.features.push(clonedFeature);
+            }
+            //strip the locations from the original feature (this should prevent any
+            //issues from the locations data contradicting the feature start/end data)
+            //feature.locations = [];
+          }
+
+          feature.start = feature.locations[0].start;
+          feature.end = feature.locations[0].end;
+        }
+        delete feature.locations;
+      });
+
+  } else {
+    //mange locations
+    oldTeselagenJson &&
+      oldTeselagenJson.features.forEach(function(feature) {
+        if (feature.locations && feature.locations[0]) {
+          //set the new starts and ends
+          feature.start = feature.locations[0].start;
+          feature.end = feature.locations[feature.locations.length - 1].end;
+          if (feature.locations.length > 1) {
+            // make sure the locations all fit within the range
+            const locationError = feature.locations.some(location => {
+              return !isRangeWithinRange(
+                location,
+                feature,
+                oldTeselagenJson.sequence.length
+              );
+            });
+            if (locationError) {
+              delete feature.locations;
+            }
+          } else {
+            delete feature.locations
+          }
+        }
+      });
+  }
+  if (Array.isArray(oldTeselagenJson.sequence)) {
+    oldTeselagenJson.sequence = oldTeselagenJson.sequence.join("");
+  }
+};

package/src/utils/createInitialSequence.js

@@ -0,0 +1,14 @@
+import { untitledSequenceName } from "./constants";
+
+export default function createInitialSequence(options) {
+    options = options || {}
+    return {
+        messages: [],
+        success: true,
+        parsedSequence: {
+            features: [],
+            name: (options.fileName && options.fileName.replace(/\.[^/.]+$/, "")) || untitledSequenceName,
+            sequence: ''
+        }
+    };
+};

package/src/utils/extractFileExtension.js

@@ -0,0 +1,14 @@
+export default function extractFileExtension(name) {
+	if (typeof name === 'string') {
+		let ext = "";
+		const match = name.match(/\.(\w+)$/);
+		if (match && match[1]) {
+			ext = match[1];
+		}
+		return ext;
+	}
+	else {
+		return "";
+	}
+};
+

package/src/utils/flattenSequenceArray.js

@@ -0,0 +1,17 @@
+import convertOldSequenceDataToNewDataType from './convertOldSequenceDataToNewDataType.js';
+
+export default function flattenSequenceArray(parsingResultArray, opts) {
+    if (parsingResultArray) {
+        if (!Array.isArray(parsingResultArray)) {
+            //wrap the parsingResult into an array if it isn't one already
+            parsingResultArray = [parsingResultArray];
+        }
+        //should convert the old data type to the new data type (flattened sequence)
+        parsingResultArray.forEach(function(parsingResult) {
+            if (parsingResult.success) {
+                convertOldSequenceDataToNewDataType(parsingResult.parsedSequence, opts);
+            }
+        });
+    }
+    return parsingResultArray;
+};

package/src/utils/getArrayBufferFromFile.js

@@ -0,0 +1,32 @@
+import { Buffer } from 'buffer';
+import isBrowser from './isBrowser';
+
+export default function getArrayBufferFromFile(file) {
+  if (!isBrowser) {
+    //node environment
+    return toArrayBuffer(Buffer.isBuffer(file) ? file : file.buffer || file);
+  }
+
+  const reader = new window.FileReader();
+  return new Promise((resolve, reject) => {
+    reader.onload = (e) => {
+      resolve(e.target.result);
+    };
+    reader.onerror = (err) => {
+      console.error('err:', err);
+      reject(err);
+    };
+    reader.readAsArrayBuffer(
+      Buffer.isBuffer(file) ? file : file.buffer || file
+    );
+  });
+}
+
+function toArrayBuffer(buffer) {
+  const ab = new ArrayBuffer(buffer.length);
+  const view = new Uint8Array(ab);
+  for (let i = 0; i < buffer.length; ++i) {
+    view[i] = buffer[i];
+  }
+  return ab;
+}

package/src/utils/isBrowser.js

@@ -0,0 +1 @@
+export default typeof window !== 'undefined' && typeof window.document !== 'undefined';

package/src/utils/parseUracilFeatures.js

@@ -0,0 +1,13 @@
+export default function parseUracilFeatures(sequenceBps, featureList = []) {
+  const cleanedBps = sequenceBps.replace(/u/gi, (u, index) => {
+    featureList.push({
+      type: "misc_feature",
+      name: "tg_uracil",
+      strand: 1,
+      start: index,
+      end: index,
+    });
+    return u === "U" ? "T" : "t";
+  });
+  return cleanedBps
+}

package/src/utils/pragmasAndTypes.js

@@ -0,0 +1,21 @@
+export default [
+  { //primers don't need a pragma because they already have a feature type of primer_bind
+    type: "primers"
+  },
+  {
+    pragma: "Teselagen_Part",
+    type: "parts"
+  },
+  {
+    pragma: "j5_warning",
+    type: "warnings"
+  },
+  {
+    pragma: "j5_assembly_piece",
+    type: "assemblyPieces"
+  },
+  {
+    pragma: "j5_lineage_annotation",
+    type: "lineageAnnotations"
+  }
+]

package/src/utils/searchWholeObjByName.js

@@ -0,0 +1,98 @@
+//tnr: taken from https://github.com/angus-c/waldojs (not using waldojs as it is not being maintained and pulled in nasty babel runtime transforms)
+/* eslint-disable eqeqeq */
+import { isEqual } from "lodash";
+
+class Match {
+  constructor(props) {
+    Object.assign(this, props);
+    this.value = this.obj[this.prop];
+  }
+
+  toString() {
+    const { path, type } = this;
+    return `${path} -> (${type}) ${this.logValue()}`;
+  }
+
+  logValue() {
+    const val = this.value;
+    // if value is an object then just toString it
+    const isPrimitive = (x) => Object(x) !== x;
+    return isPrimitive(val) || Array.isArray(val) ? val : {}.toString.call(val);
+  }
+
+  log() {
+    console.info(this.toString());
+  }
+}
+
+const GLOBAL = typeof window == "object" ? window : global;
+
+export default function searchWholeObjByName(what, where) {
+  const searchBy = (what, where, prop) => what == prop;
+
+  let data;
+  let alreadySeen;
+
+  const path = where == GLOBAL ? "GLOBAL" : "SRC";
+  const queue = [{ where, path }];
+  const seen = [];
+
+  const matches = [];
+  matches.log = function () {
+    this.forEach((m) => m.log());
+  };
+
+  // a non-recursive solution to avoid call stack limits
+  // http://www.jslab.dk/articles/non.recursive.preorder.traversal.part4
+  while ((data = queue.pop())) {
+    const { where, path } = data;
+
+    for (const prop in where) {
+      // IE may throw errors when accessing/coercing some properties
+      try {
+        // eslint-disable-next-line no-prototype-builtins
+        if (where.hasOwnProperty(prop)) {
+          // inspect objects
+          if ([where[prop]] == "[object Object]") {
+            // check if already searched (prevents circular references)
+            for (
+              let i = -1;
+              seen[++i] &&
+              !(alreadySeen = isEqual(seen[i].where, where[prop]) && seen[i]);
+
+            );
+            // add to stack
+            if (!alreadySeen) {
+              data = { where: where[prop], path: `${path}.${prop}` };
+              queue.push(data);
+              seen.push(data);
+            }
+          }
+          // if match detected, push it.
+          if (searchBy(what, where, prop)) {
+            const type = alreadySeen
+              ? `<${alreadySeen.path}>`
+              : typeof where[prop];
+            const match = new Match({
+              path: `${path}.${prop}`,
+              obj: where,
+              prop,
+              type,
+            });
+            matches.push(match);
+          }
+        }
+      } catch (e) {
+        // don't throw errs
+      }
+    }
+  }
+
+  return matches;
+}
+export function searchWholeObjByNameSimple(what, where) {
+  return searchWholeObjByName(what, where)?.[0]?.value?.[0];
+}
+export function searchWholeObjByNameSimpleArray(what, where) {
+  return searchWholeObjByName(what, where)?.[0]?.value;
+}

package/src/utils/splitStringIntoLines.js

@@ -0,0 +1,13 @@
+export default function splitStringIntoLines(string) {
+	let lines = [];
+	if (string === "") {
+		return lines;
+	}
+	else {
+		lines = string.split(/\r?\n/);
+		if (lines.length == 1) { //tnr: not sure why this check is being made... but keeping it in because it is probably doing something
+			lines = string.split('\\n');
+		}
+		return lines;
+	}
+};