@figtreejs/core 0.0.1-alpha.0

package/src/evo/tree/parsers/parsing.test.ts

@@ -0,0 +1,289 @@
+import { u } from "../../../utils";
+import { ImmutableTree } from "../normalized-tree/immutable-tree";
+import { processAnnotationValue } from "./annotation-parser";
+import { describe, it, expect } from "vitest";
+describe("Test annotation parsing", () => {
+  it("integer", function () {
+    const annotation = processAnnotationValue("1");
+    expect(annotation).toEqual({ type: "NUMERICAL", value: 1 });
+  });
+  it("integer array", function () {
+    const annotation = processAnnotationValue(["1", "2", "3"]);
+    expect(annotation).toEqual({ type: "NUMERICAL_SET", value: [1, 2, 3] });
+  });
+
+  it("continuous array", function () {
+    const annotation = processAnnotationValue(["1", "2.5", "3"]);
+    expect(annotation).toEqual({ type: "NUMERICAL_SET", value: [1, 2.5, 3] });
+  });
+
+  it("continuous range", function () {
+    const annotation = processAnnotationValue([0.0, 1]);
+    expect(annotation).toEqual({ type: "NUMERICAL_SET", value: [0, 1] });
+  });
+
+  it("continuous", function () {
+    const annotation = processAnnotationValue("1.3");
+    expect(annotation).toEqual({ type: "NUMERICAL", value: 1.3 });
+  });
+  it("discrete", function () {
+    const annotation = processAnnotationValue("a");
+    expect(annotation).toEqual({ type: "DISCRETE", value: "a" });
+  });
+  it("discrete array", function () {
+    const annotation = processAnnotationValue(["a", "b", "c"]);
+    expect(annotation).toEqual({
+      type: "DISCRETE_SET",
+      value: ["a", "b", "c"],
+    });
+  });
+  it("markov jump", function () {
+    const annotation = processAnnotationValue([["0.1", "U", "me"]]);
+    expect(annotation).toEqual({
+      type: "MARKOV_JUMPS",
+      value: [{ time: 0.1, from: "U", to: "me" }],
+    });
+  });
+  it("markov jump array", function () {
+    const annotation = processAnnotationValue([
+      ["0.1", "U", "me"],
+      ["0.2", "me", "U"],
+    ]);
+    expect(annotation).toEqual({
+      type: "MARKOV_JUMPS",
+      value: [
+        { time: 0.1, from: "U", to: "me" },
+        { time: 0.2, from: "me", to: "U" },
+      ],
+    });
+  });
+  it("probabilities", function () {
+    const annotation = processAnnotationValue({ HERE: 0.3, THERE: 0.7 });
+    expect(annotation).toEqual({
+      type: "DENSITIES",
+      value: { HERE: 0.3, THERE: 0.7 },
+    });
+  });
+});
+
+describe("Test tree parsing and normalized Tree", () => {
+  it("simpleParse", function () {
+    const newickString = `((((((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2,(virus6:0.45,virus7:0.4):0.02):0.1,virus8:0.4):0.1,(virus9:0.04,virus10:0.03):0.6);`;
+
+    const tree = ImmutableTree.fromNewick(newickString, {
+      parseAnnotations: false,
+      labelName: "probability",
+    });
+    expect(tree.toNewick()).toEqual(newickString);
+  });
+
+  it("height", function () {
+    const newickString = `((((((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2,(virus6:0.45,virus7:0.4):0.02):0.1,virus8:0.4):0.1,(virus9:0.04,virus10:0.03):0.6);`;
+    const tree = ImmutableTree.fromNewick(newickString, { labelName: "prob" });
+    const virus1 = tree.getTaxonByName("virus1");
+    const virus1Node = tree.getNodeByTaxon(virus1);
+    expect(tree.getHeight(virus1Node)).toBeCloseTo(0.06, 1e-6);
+  });
+  it("divergence", function () {
+    const newickString = `((((((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2,(virus6:0.45,virus7:0.4):0.02):0.1,virus8:0.4):0.1,(virus9:0.04,virus10:0.03):0.6);`;
+    const tree = ImmutableTree.fromNewick(newickString, { labelName: "prob" });
+    const virus6 = tree.getTaxonByName("virus6");
+    const virus6Node = tree.getNodeByTaxon(virus6);
+    expect(tree.getDivergence(virus6Node)).toBeCloseTo(0.67, 1e-6);
+  });
+  it("general_parse", function () {
+    const tree = ImmutableTree.fromNewick("(a:1,b:4)#l;");
+    const root = tree.getRoot();
+    const label = tree.getLabel(root);
+    expect(label).toEqual("l");
+
+    const r = tree.getNodeByLabel("l");
+    expect(r).toEqual(root);
+
+    const names = [];
+    const bl = [];
+
+    const count = tree.getChildCount(root);
+
+    for (let i = 0; i < count; i++) {
+      const child = tree.getChild(root, i);
+      names.push(tree.getTaxonFromNode(child).name);
+      bl.push(tree.getLength(child));
+    }
+    expect(names).toEqual(["a", "b"]);
+
+    expect(bl).toEqual([1, 4]);
+  });
+
+  it("scientific notation", function () {
+    const tree = ImmutableTree.fromNewick("(a:1E1,b:2e-5);");
+    const root = tree.getRoot();
+    const bl = [];
+    const count = tree.getChildCount(root);
+
+    for (let i = 0; i < count; i++) {
+      const child = tree.getChild(root, i);
+      bl.push(tree.getLength(child));
+    }
+    expect(bl[0]).toBeCloseTo(10.0, 1e-6);
+    expect(bl[1]).toBeCloseTo(0.00002, 1e-6);
+  });
+
+  it("quoted taxa", function () {
+    const tree = ImmutableTree.fromNewick("('234] ':1,'here a *':1);");
+    const names = [...tree.getExternalNodes()].map(
+      (node) => tree.getTaxonFromNode(node).name,
+    );
+    expect(names).toEqual(["234]", "here a *"]);
+  });
+
+  it("whitespace", function () {
+    const tree = ImmutableTree.fromNewick("  (a,b:1);\t");
+    expect(tree.toNewick()).toEqual("(a,b:1);");
+  });
+  it("node id", function () {
+    const tree = ImmutableTree.fromNewick("((A,T)#Node_1:1,(a,b:1));");
+    const A = tree.getTaxonByName("A");
+    const B = tree.getTaxonByName("b");
+    const Anode = tree.getNodeByTaxon(A);
+    const node1 = tree.getNodeByLabel("Node_1");
+    const parent = tree.getParent(Anode);
+
+    expect(parent).toEqual(node1);
+
+    expect(tree.getLength(u(tree.getNodeByTaxon(B)))).toEqual(1);
+    expect(tree.toNewick()).toEqual("((A,T)#Node_1:1,(a,b:1));");
+  });
+  it("root length and label", function () {
+    const tree = ImmutableTree.fromNewick("((A,T)#Node_1:1,(a,b:1))#root:0.1;");
+    const root = tree.getRoot();
+    const rootLength = tree.getLength(root);
+    expect(rootLength).toEqual(0.1);
+    const label = tree.getLabel(root);
+    expect(label).toEqual("root");
+    expect(tree.toNewick()).toEqual("((A,T)#Node_1:1,(a,b:1))#root:0.1;");
+  });
+
+  it("fail no ;", function () {
+    expect(() => ImmutableTree.fromNewick("('234] ','here a *')")).toThrow(
+      "expecting a semi-colon at the end of the newick string",
+    );
+  });
+
+  it("fail unbalanced )", function () {
+    expect(() => ImmutableTree.fromNewick("(a,b));")).toThrow(
+      "the brackets in the newick file are not balanced: too many closed",
+    );
+  });
+
+  it("fail unbalanced (", function () {
+    expect(() => ImmutableTree.fromNewick("((a,b);")).toThrow(
+      "unexpected semi-colon in tree did not reach the root yet",
+    );
+  });
+
+  it("comment", function () {
+    const tree = ImmutableTree.fromNewick("(a[&test=ok],b:1);", {
+      parseAnnotations: true,
+    });
+    const a = tree.getTaxonByName("a");
+    const aNode = tree.getNodeByTaxon(a);
+    const testAnnotation = tree.getAnnotation(aNode, "test");
+    expect(testAnnotation).toEqual("ok");
+  });
+
+  it("markov jump comment", function () {
+    const tree = ImmutableTree.fromNewick(
+      "(a[&test=ok],b[&jump={{0.1,U,me}}]);",
+      { parseAnnotations: true },
+    );
+    const a = tree.getNodeByTaxon(tree.getTaxonByName("a"));
+    const b = tree.getNodeByTaxon(tree.getTaxonByName("b"));
+    const testAnnotation = tree.getAnnotation(a, "test");
+    expect(testAnnotation).toEqual("ok");
+    const jumpAnnotation = tree.getAnnotation(b, "jump");
+    expect(jumpAnnotation).toEqual([{ time: 0.1, from: "U", to: "me" }]);
+  });
+
+  it("double comment", function () {
+    const tree = ImmutableTree.fromNewick("(a[&test=ok,other test = 1],b:1);", {
+      parseAnnotations: true,
+    });
+    const a = tree.getNodeByTaxon(tree.getTaxonByName("a"));
+    const testAnnotation = tree.getFullNodeAnnotation(a, "test");
+
+    expect(testAnnotation).toEqual({
+      id: "test",
+      type: "DISCRETE",
+      value: "ok",
+    });
+    const otherTestAnnotation = tree.getFullNodeAnnotation(a, "other test");
+    expect(otherTestAnnotation).toEqual({
+      id: "other test",
+      type: "NUMERICAL",
+      value: 1,
+    });
+  });
+
+  it("SET comment", function () {
+    const tree = ImmutableTree.fromNewick(
+      "(a[&test={ok,Not ok},other test = {0,1}],b:1);",
+      {
+        parseAnnotations: true,
+      },
+    );
+    const a = tree.getNodeByTaxon(tree.getTaxonByName("a"));
+    const testAnnotation = tree.getFullNodeAnnotation(a, "test");
+    expect(testAnnotation).toEqual({
+      id: "test",
+      type: "DISCRETE_SET",
+      value: ["ok", "Not ok"],
+    });
+
+    const otestAnnotation = tree.getFullNodeAnnotation(a, "other test");
+    expect(otestAnnotation).toEqual({
+      id: "other test",
+      type: "NUMERICAL_SET",
+      value: [0, 1],
+    });
+  });
+
+  it("SET comment writing", function () {
+    const treeString = "(a[&test={ok, Not ok}, other test={0, 1}],b:1);";
+    const tree = ImmutableTree.fromNewick(treeString, {
+      parseAnnotations: true,
+    });
+    expect(tree.toNewick(undefined, { includeAnnotations: true })).toEqual(
+      treeString,
+    );
+  });
+  it("label annotation", function () {
+    const tree = ImmutableTree.fromNewick(
+      "((((((virus1:0.1,virus2:0.12)0.95:0.08,(virus3:0.011,virus4:0.0087)1.0:0.15)0.65:0.03,virus5:0.21)1.0:0.2,(virus6:0.45,virus7:0.4)0.51:0.02)1.0:0.1,virus8:0.4)1.0:0.1,(virus9:0.04,virus10:0.03)1.0:0.6);",
+      { parseAnnotations: true, labelName: "probability" },
+    );
+    const virus1Node = tree.getNodeByTaxon(tree.getTaxonByName("virus1"));
+
+    const probability = tree.getAnnotation(
+      tree.getParent(virus1Node),
+      "probability",
+    );
+
+    expect(probability).toEqual(0.95);
+  });
+  it("nexus", function () {
+    const nexusString = `#NEXUS
+BEGIN TAXA;
+ DIMENSIONS NTAX=10;
+ TAXLABELS virus1 virus10 virus2 virus3 virus4 virus5 virus6 virus7 virus8 virus9;
+END;
+BEGIN TREES;
+  TREE tree0 = ((((((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2,(virus6:0.45,virus7:0.4):0.02):0.1,virus8:0.4):0.1,(virus9:0.04,virus10:0.03):0.6);
+END;
+`;
+    const tree = ImmutableTree.fromNexus(nexusString);
+    const newickString = `((((((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2,(virus6:0.45,virus7:0.4):0.02):0.1,virus8:0.4):0.1,(virus9:0.04,virus10:0.03):0.6);`;
+
+    expect(tree.toNewick()).toEqual(newickString);
+  });
+});

package/src/evo/tree/parsers/stream-reader/index.ts

@@ -0,0 +1 @@
+export * from "./nexus-importer";

package/src/evo/tree/parsers/stream-reader/newick-importer.txt

@@ -0,0 +1,395 @@
+import { ImmutableTree } from "../../NormalizedTree"
+import { TaxonSet } from "../../Taxa/Taxon"
+
+class newickImporter {
+  reader: ReadableStreamDefaultReader<string>
+  utf8Decoder: TextDecoder
+  taxonSet: TaxonSet | undefined
+  currentBlock: string | undefined
+  hasTree: boolean | undefined
+  options: { labelName?: string }
+
+//TODO update the options here to just read in trees.
+
+
+  constructor(
+    stream: ReadableStream<Uint8Array>,
+    options: { labelName?: string } = {},
+  ) {
+    this.utf8Decoder = new TextDecoder("utf-8")
+    const newickTransformer = new TransformStream(transformerOptions())
+
+    this.reader = stream
+      .pipeThrough(new TextDecoderStream())
+      .pipeThrough(newickTransformer)
+      .getReader()
+
+    this.taxonSet = undefined
+    this.currentBlock = undefined
+    this.options = options
+  }
+
+  async *getNextTree(taxa:TaxonSet): AsyncIterableIterator<ImmutableTree> {
+    yield* this.parseNextTree()
+  }
+
+  async parseNextBlock() {
+    const blockName = await this.getNextBlockName()
+    switch (blockName) {
+      case "taxa":
+        this.currentBlock = "taxa"
+        await this.parseTaxaBlock()
+        break
+      case "trees":
+        this.currentBlock = "trees"
+        break
+      default:
+        console.log(
+          `skipping block ${blockName}. Only parsing blocks are taxa and trees for now.`,
+        )
+        await this.readToEndOfBlock()
+    }
+  }
+
+  async nextToken() {
+    const { done, value } = await this.reader.read()
+    if (done) {
+      throw "unexpectedly hit the end of the stream"
+    }
+    return value
+  }
+  //make skip and then a list of regex to skip
+  async skipSemiColon() {
+    const { done, value } = await this.reader.read()
+    if (done) {
+      throw "unexpectedly hit the end of the stream"
+    }
+    if (!value.match(/;$/)) {
+      throw `expected ";" got ${value}`
+    }
+  }
+
+  async getNextBlockName() {
+    while (true) {
+      const value = await this.nextToken()
+
+      if (value.match(/\bbegin/i)) {
+        const token = await this.nextToken()
+        this.skipSemiColon()
+        return token
+      }
+    }
+  }
+
+  async readToEndOfBlock() {
+    while (true) {
+      const value = await this.nextToken()
+      if (value.match(/\bend;/i)) {
+        break
+      }
+    }
+  }
+
+  async getNextCommand(command: RegExp) {
+    while (true) {
+      const value = await this.nextToken()
+      if (value === ";") {
+        throw `Hit ; looking for ${command}`
+      }
+      if (command.test(value)) {
+        return value
+        break
+      }
+    }
+  }
+  // skip until match and return match
+  async skipUntil(stopper: RegExp) {
+    while (true) {
+      const value = await this.nextToken()
+      if (stopper.test(value)) {
+        return value
+      }
+    }
+  }
+  // read up to match return everything up to including the match
+  async readUntil(stopper: RegExp) {
+    let buffer = ""
+    while (true) {
+      const value = await this.nextToken()
+      if (stopper.test(value)) {
+        return buffer + value
+      }
+      buffer += value
+    }
+  }
+  async parseTaxaBlock() {
+    let ntax
+    let command = await this.skipUntil(/dimensions|taxlabels|end/i)
+    while (true) {
+      switch (true) {
+        case /dimensions/i.test(command):
+          const taxaLine = await this.readUntil(/;/)
+          const ntaxa = taxaLine.match(/ntax=(\d+);/)
+          if (ntaxa) {
+            ntax = parseInt(ntaxa[1])
+          } else {
+            throw `Expected dimension in form of ntax=(\d+);. Got ${taxaLine}`
+          }
+          break
+        case /taxlabels/i.test(command):
+          let token = await this.nextToken()
+          this.taxonSet = new TaxonSet()
+          while (token !== ";") {
+            this.taxonSet.addTaxon(token)
+            token = await this.nextToken()
+          }
+          if (ntax) {
+            if (ntax != this.taxonSet.getTaxonCount()) {
+              throw `found ${this.taxonSet.getTaxonCount()} taxa. Expected: ${ntax}}`
+            }
+          }
+          break
+        case /end/i.test(command):
+          if (this.taxonSet!.getTaxonCount() === 0) {
+            throw "hit end of taxa section but didn't find any taxa"
+          }
+          this.skipSemiColon()
+        default:
+          throw `Reached impossible code looking for dimensions or taxlabels in taxa block "${command}"`
+      }
+      command = await this.skipUntil(/dimensions|taxlabels|end/i)
+    }
+  }
+
+  async *parseNextTree() {
+    let command = await this.skipUntil(/translate|tree|end/i)
+    let token
+    while (true) {
+      switch (true) {
+        case /translate/i.test(command):
+          // all white space removed by tranformStream so will be
+          // ['key','taxon,'] but may be ['key','taxon',','] if space tween taxa and ,
+          this.translateTaxonMap = new Map()
+          let newTaxonSet = false
+          if (!this.taxonSet) {
+            this.taxonSet = new TaxonSet()
+            newTaxonSet = true
+          }
+          let i = 0
+          let key
+          token = await this.nextToken()
+          while (token !== ";") {
+            if (i % 2 == 0) {
+              key = token
+            } else {
+              if (token[token.length - 1] === ",") {
+                token = token.slice(0, -1)
+              }
+              // todo get taxa to add here.
+              if (!newTaxonSet) {
+                if (this.taxonSet.getTaxonByName(token) === undefined) {
+                  throw `Taxon ${token} not found in taxa block - but found in translate block`
+                }
+              } else {
+                //new taxon set so add it;
+                this.taxonSet.addTaxon(token)
+              }
+              const taxon = this.taxonSet.getTaxonByName(token)
+              this.translateTaxonMap.set(key!, taxon)
+            }
+            token = await this.nextToken()
+            while (token === ",") {
+              token = await this.nextToken()
+            } //incase some white space in there
+            i++
+          }
+          break
+        case /tree/i.test(command):
+          //parse tree
+          // put this in loop so the next call parses the next tree;
+
+          // first token will be tree id
+          // then possible annotation
+          // Then =
+          // then possible annotations
+          // then tree
+          const treeId = await this.nextToken()
+          // read to first '(';
+          token = await this.skipUntil(/\(/)
+          let buffer = token
+            .split(newickDeliminators)
+            .filter((n) => n.length > 0)
+            .reverse()
+
+          let level = 0
+          let currentNode: NodeRef | undefined = undefined
+          let nodeStack: NodeRef[] = []
+          let labelNext = false
+          let lengthNext = false
+
+          let tree = new ImmutableTree({ taxonSet: this.taxonSet })
+
+          while (token !== ";") {
+            while (buffer.length > 0) {
+              const t = buffer.pop()!
+              if (t.length > 2 && t.substring(0, 2) === "[&") {
+                const annotations = parseAnnotation(token)
+
+                tree = tree.annotateNode(currentNode!, annotations)
+              } else if (token === "(") {
+                // an internal node
+
+                if (labelNext) {
+                  // if labelNext is set then the last bracket has just closed
+                  // so there shouldn't be an open bracket.
+                  throw new Error("expecting a comma")
+                }
+                let node
+                level += 1
+                if (currentNode !== undefined) {
+                  const added = tree.addNodes(1)
+                  tree = added.tree
+                  node = added.nodes[0]
+                  nodeStack.push(currentNode)
+                } else {
+                  node = tree.getRoot()
+                }
+                currentNode = node
+              } else if (token === ",") {
+                // another branch in an internal node
+
+                labelNext = false // labels are optional
+                if (lengthNext) {
+                  throw new Error("branch length missing")
+                }
+
+                let parent = nodeStack.pop()! as NodeRef
+                tree = tree.addChild(parent, currentNode!)
+                // tree.setParent(currentNode!,parent)
+
+                currentNode = parent
+              } else if (token === ")") {
+                if (level === 0) {
+                  throw new Error(
+                    "the brackets in the newick file are not balanced: too many closed",
+                  )
+                }
+                // finished an internal node
+
+                labelNext = false // labels are optional
+                if (lengthNext) {
+                  throw new Error("branch length missing")
+                }
+
+                // the end of an internal node
+                let parent = nodeStack.pop()! as NodeRef
+                tree = tree.addChild(parent, currentNode!)
+                // tree.setParent(currentNode!,parent)
+
+                level -= 1
+                currentNode = parent
+
+                labelNext = true
+              } else if (token === ":") {
+                labelNext = false // labels are optional
+                lengthNext = true
+              } else {
+                // not any specific token so may be a label, a length, or an external node name
+                if (lengthNext) {
+                  tree = tree.setLength(currentNode!, parseFloat(token))
+                  lengthNext = false
+                } else if (labelNext) {
+                  if (!token.startsWith("#")) {
+                    let value: number | any = parseFloat(token)
+                    if (isNaN(value)) {
+                      value = token
+                    }
+                    if (this.options.labelName) {
+                      let label_annotation = {
+                        name: this.options.labelName,
+                        value: value,
+                      }
+                      tree = tree.annotateNode(currentNode!, label_annotation)
+                    } else {
+                      console.warn(
+                        `No label name provided to newick parser but found label ${token}. It will be ignored`,
+                      )
+                    }
+                  } else {
+                    tree = tree.setLabel(currentNode!, token.slice(1)) //remove the # todo put it back when writing to newick
+                  }
+                  labelNext = false
+                } else {
+                  let name = token // TODO tree needs be a map that's not the ID
+
+                  // remove any quoting and then trim whitespace
+                  // TODO add to bit that parses taxa block
+                  if (name.startsWith('"') || name.startsWith("'")) {
+                    name = name.substr(1)
+                  }
+                  if (name.endsWith('"') || name.endsWith("'")) {
+                    name = name.substr(0, name.length - 1)
+                  }
+                  name = name.trim()
+
+                  const added = tree.addNodes(1)
+                  tree = added.tree
+                  const externalNode = added.nodes[0]
+                  let taxon: Taxon
+                  if (this.translateTaxonMap) {
+                    if (this.translateTaxonMap.has(name)) {
+                      taxon = this.translateTaxonMap.get(name)!
+                    } else {
+                      throw `No mapping found for ${name} in tipNameMap. It's name will not be updated`
+                    }
+                  } else if (this.taxonSet) {
+                    taxon = this.taxonSet.getTaxonByName(name)
+                    if (taxon === undefined) {
+                      throw `Taxon ${name} not found in taxa - but found in tree`
+                    }
+                  } else {
+                    tree.addTaxon(name) //does this affect immutability - using the first tree as a taxon set.
+                    taxon = tree.getTaxonByName(name)!
+                  }
+
+                  tree = tree.setTaxon(externalNode, taxon)
+
+                  if (currentNode) {
+                    nodeStack.push(currentNode)
+                  }
+                  currentNode = externalNode
+                }
+              }
+            }
+            // get next token from reader
+            token = await this.nextToken()
+            buffer = token
+              .split(newickDeliminators)
+              .filter((n) => n.length > 0)
+              .reverse()
+          }
+          if (level > 0) {
+            throw new Error(`unexpected semi-colon in tree: ${treeId}`)
+          }
+          if (this.taxonSet === undefined) {
+            this.taxonSet === tree
+          }
+          yield tree
+          break
+        case /end/i.test(command):
+          this.skipSemiColon()
+          this.hasTree = false
+          // Give up the reader.
+          //TODO read to the end of the file.
+          this.reader.releaseLock()
+          break
+        default:
+          throw `Reached impossible code in treeblock block "${command}"`
+      }
+      command = await this.skipUntil(/translate|tree|end/i)
+    }
+  }
+}
+
+
+