@figtreejs/core 0.0.1-alpha.0

package/src/evo/tree/parsers/stream-reader/nexus-importer.test.ts

@@ -0,0 +1,99 @@
+import { ImmutableTree } from "../../normalized-tree";
+import { NexusImporter } from "./nexus-importer";
+import { describe, it, expect } from "vitest";
+function stringToReadableStream(str: string) {
+  return new ReadableStream({
+    start(controller: {
+      enqueue: (arg0: Uint8Array<ArrayBuffer>) => void;
+      close: () => void;
+    }) {
+      // Convert the string to a Uint8Array and enqueue it
+      const encoder = new TextEncoder();
+      const chunk = encoder.encode(str);
+      controller.enqueue(chunk);
+      controller.close();
+    },
+  });
+}
+
+describe("Testing nexus importer", () => {
+  it("parse  Nexus", async function () {
+    const nexusString = `#NEXUS
+        Begin trees;
+        tree tree1 = (((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2;
+        End;`;
+
+    const nexusStream = stringToReadableStream(nexusString);
+
+    const trees = new NexusImporter(nexusStream).getTrees();
+    let i = 0;
+    for await (const tree of trees) {
+      expect(tree).toBeInstanceOf(ImmutableTree);
+      expect(tree.getExternalNodeCount()).toBe(5);
+      expect(tree.getInternalNodeCount()).toBe(4);
+      i++;
+    }
+    expect(i).toBe(1);
+  });
+  it("parse  Nexus multiple trees", async function () {
+    const nexusString = `#NEXUS
+      Begin trees;
+      tree tree1 = (((virus1:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2;
+      tree tree2 = (((Tip7:0.04873764378778993,(Tip8:0.6532000878000133,Tip6:0.1321456687110038):0.042190933211748174):0.6632395745455513,Tip5:0.07721466870767527):0.023945361546555622,Tip0:0.013537352040642556,(Tip3:0.13083726861861736,(((Tip9:0.048159442284509385,Tip4:0.0727658295569952):0.03481552263500266,Tip2:0.0821610870291415):0.13059030569905986,Tip1:0.03707788908419223):0.04920278810640517):0.0011230070183312782);
+      End;`;
+
+    const nexusStream = stringToReadableStream(nexusString);
+
+    const trees = new NexusImporter(nexusStream).getTrees();
+    let i = 0;
+    let lastTree;
+    for await (const tree of trees) {
+      expect(tree).toBeInstanceOf(ImmutableTree);
+      lastTree = tree;
+      i++;
+    }
+    expect(i).toBe(2);
+    expect(lastTree?.getTaxonCount()).toBe(15); //has taxa available from all trees
+  });
+  it("parse  Nexus - treeannotations", async function () {
+    const nexusString = `#NEXUS
+    Begin trees;
+    tree tree1 = (((virus1[&test="A",height=90]:0.1,virus2:0.12):0.08,(virus3:0.011,virus4:0.0087):0.15):0.03,virus5:0.21):0.2;
+    End;`;
+
+    const nexusStream = stringToReadableStream(nexusString);
+
+    const trees = new NexusImporter(nexusStream).getTrees();
+    let i = 0;
+    for await (const tree of trees) {
+      expect(tree).toBeInstanceOf(ImmutableTree);
+      i++;
+    }
+    expect(i).toBe(1);
+  });
+  it("parse  Nexus multiple trees, taxa", async function () {
+    const nexusString = `#NEXUS
+    Begin taxa;
+      Dimensions ntax=5;
+      TaxLabels Tip0 Tip1 Tip2 Tip3 Tip4;
+    End;
+    Begin trees;
+      tree 1 = ((Tip3:0.005683055909367637,(Tip4:0.028897909268250865,Tip2:0.17725578701486128):0.15953222892967397):0.0015945469810341964,Tip0:0.16482157194489672,Tip1:0.06558413486573018);
+      tree 2 = (Tip3:0.07895643336574805,Tip0:0.09039621608818542,(Tip2:0.12447554684923097,(Tip4:0.016569217859539333,Tip1:0.04529906914098861):0.4270260158852053):0.5512862407921115);
+      tree 3 = ((Tip4:0.15573112107230094,Tip2:0.10434839467828654):0.10910110483363722,Tip0:0.009239519001625887,(Tip3:0.4614463596422172,Tip1:0.087294319045952):0.22603324870135144);
+    End;`;
+
+    const nexusStream = stringToReadableStream(nexusString);
+
+    const trees = new NexusImporter(nexusStream).getTrees();
+    let i = 0;
+    let lastTree;
+    for await (const tree of trees) {
+      expect(tree).toBeInstanceOf(ImmutableTree);
+      lastTree = tree;
+      i++;
+    }
+    expect(i).toBe(3);
+    expect(lastTree?.getTaxonCount()).toBe(5); //has taxa available from all trees
+  });
+});

package/src/evo/tree/parsers/stream-reader/nexus-importer.ts

@@ -0,0 +1,293 @@
+import { notNull } from "../../../../utils";
+import type { ImmutableTree } from "../../normalized-tree";
+import { TaxonSet } from "../../taxa/taxon";
+
+import { NewickCharacterParser } from "../newick-character-parser";
+import { newickDeliminators, nexusTokenizer } from "./nexus-tokenizer";
+
+export class NexusImporter {
+  reader: ReadableStreamDefaultReader<string>;
+  taxonSet: TaxonSet;
+  currentBlock: string | undefined;
+  hasTree: boolean | undefined;
+  options: { labelName?: string };
+  translateTaxonMap: Map<string, string> | undefined;
+  constructor(
+    stream: ReadableStream<BufferSource>,
+    options: { labelName?: string } = {},
+  ) {
+    const tokenizer = new TransformStream(nexusTokenizer());
+
+    this.reader = stream
+      .pipeThrough(new TextDecoderStream())
+      .pipeThrough(tokenizer)
+      .getReader();
+
+    this.taxonSet = new TaxonSet();
+    this.currentBlock = undefined;
+    this.options = options;
+  }
+
+  async *getTrees(): AsyncIterableIterator<ImmutableTree> {
+    while (this.currentBlock !== "trees") {
+      await this.parseNextBlock();
+      this.hasTree = true;
+    }
+    yield* this.parseTreesBlock();
+  }
+
+  private async parseNextBlock() {
+    const blockName = await this.getNextBlockName();
+    switch (blockName) {
+      case "taxa":
+        this.currentBlock = "taxa";
+        await this.parseTaxaBlock();
+        break;
+      case "trees":
+        this.currentBlock = "trees";
+        break;
+      default:
+        console.log(
+          `skipping block ${blockName}. Only parsing blocks are taxa and trees for now.`,
+        );
+        await this.readToEndOfBlock();
+    }
+  }
+
+  private async nextToken() {
+    const { done, value } = await this.reader.read();
+    if (done) {
+      throw new Error("unexpectedly hit the end of the stream");
+    }
+    return value;
+  }
+  //make skip and then a list of regex to skip
+  private async skipSemiColon() {
+    const { done, value } = await this.reader.read();
+    if (done) {
+      throw new Error("unexpectedly hit the end of the stream");
+    }
+    if (!value.match(/;$/)) {
+      throw new Error(`expected ";" got ${value}`);
+    }
+  }
+
+  private async getNextBlockName() {
+    let keepGoing = true;
+    let token;
+    while (keepGoing) {
+      const value = await this.nextToken();
+      if (value.match(/\bbegin/i)) {
+        token = await this.nextToken();
+        await this.skipSemiColon();
+        keepGoing = false;
+      }
+    }
+    return token;
+  }
+
+  private async readToEndOfBlock() {
+    let keepGoing = true;
+    while (keepGoing) {
+      const value = await this.nextToken();
+      if (value.match(/\bend;/i)) {
+        keepGoing = false;
+      }
+    }
+  }
+
+  // private async getNextCommand(command: RegExp) {
+  //   let value;
+  //   let keepGoing = true
+  //   while (keepGoing) {
+  //     value = await this.nextToken()
+  //     if (value === ";") {
+  //       throw `Hit ; looking for ${command}`
+  //     }
+  //     if (command.test(value)) {
+  //       keepGoing=false
+  //     }
+  //   }
+  //   return value
+  // }
+  // skip until match and return match
+  private async skipUntil(stopper: RegExp): Promise<string> {
+    let value;
+    let keepGoing = true;
+    while (keepGoing) {
+      value = await this.nextToken();
+      if (stopper.test(value)) {
+        keepGoing = false;
+      }
+    }
+    if (value == undefined)
+      throw new Error(`Internal parsing error: ${stopper.source} not found `);
+    return value;
+  }
+  // read up to match return everything up to including the match
+  private async readUntil(stopper: RegExp) {
+    let buffer = "";
+    let keepGoing = true;
+    while (keepGoing) {
+      const value = await this.nextToken();
+      if (stopper.test(value)) {
+        buffer += value;
+        keepGoing = false;
+      }
+      buffer += value;
+    }
+    return buffer;
+  }
+  private async parseTaxaBlock() {
+    let ntax;
+    let keepGoing = true;
+    while (keepGoing) {
+      const command = await this.skipUntil(/dimensions|taxlabels|end/i);
+      switch (true) {
+        case /dimensions/i.test(command): {
+          const taxaLine = await this.readUntil(/;/);
+          const ntaxa = taxaLine.match(/ntax=(\d+);/);
+          if (ntaxa) {
+            notNull(ntaxa[1], `No number of taxa found despite matching regex`);
+            ntax = parseInt(ntaxa[1]);
+          } else {
+            throw new Error(
+              `Expected dimension in form of ntax=(\\d+);. Got ${taxaLine}`,
+            );
+          }
+          break;
+        }
+        case /taxlabels/i.test(command): {
+          let token = await this.nextToken();
+          while (token !== ";") {
+            this.taxonSet.addTaxon(token);
+            token = await this.nextToken();
+          }
+          if (ntax) {
+            if (ntax != this.taxonSet.getTaxonCount()) {
+              throw new Error(
+                `found ${this.taxonSet.getTaxonCount()} taxa. Expected: ${ntax}}`,
+              );
+            }
+          }
+          break;
+        }
+        case /end/i.test(command): {
+          if (this.taxonSet.getTaxonCount() === 0) {
+            throw new Error("hit end of taxa section but didn't find any taxa");
+          }
+          this.taxonSet.lockTaxa(); // no more taxa can be added since we parsed a block;
+          await this.skipSemiColon();
+          keepGoing = false;
+          break;
+        }
+        default:
+          throw new Error(
+            `Reached impossible code looking for dimensions or taxlabels or end in taxa block "${command}"`,
+          );
+      }
+    }
+  }
+
+  private async *parseTreesBlock() {
+    let token;
+    let keepGoing = true;
+    while (keepGoing) {
+      const command = await this.skipUntil(/translate|tree|end/i);
+      switch (true) {
+        case /translate/i.test(command): {
+          // all white space removed by tranformStream so will be
+          // ['key','taxon,'] but may be ['key','taxon',','] if space tween taxa and ,
+          this.translateTaxonMap = new Map();
+          let i = 0;
+          let key;
+          token = await this.nextToken();
+          while (token !== ";") {
+            if (i % 2 == 0) {
+              key = token;
+            } else {
+              if (token[token.length - 1] === ",") {
+                token = token.slice(0, -1);
+              }
+              // todo get taxa to add here.
+              if (this.taxonSet.isFinalized) {
+                if (!this.taxonSet.hasTaxon(token)) {
+                  throw new Error(
+                    `Taxon ${token} not found in taxa block - but found in translate block`,
+                  );
+                }
+              } else {
+                //new taxon set so add it;
+                this.taxonSet.addTaxon(token);
+              }
+              // const taxon = this.taxonSet.getTaxonByName(token)
+              notNull(
+                key,
+                "Error parsing nexus. Expected key for taxa but found nothing",
+              );
+              this.translateTaxonMap.set(key, token);
+            }
+            token = await this.nextToken();
+            while (token === ",") {
+              token = await this.nextToken();
+            } //incase some white space in there
+            i++;
+          }
+          this.taxonSet.lockTaxa();
+          break;
+        }
+        case /tree/i.test(command): {
+          //parse tree
+          // put this in loop so the next call parses the next tree;
+
+          // first token will be tree id
+          // then possible annotation
+          // Then =
+          // then possible annotations
+          // then tree
+          //tree id =
+          await this.nextToken(); //todo - read to'=' not just next token
+          const parser = new NewickCharacterParser(this.taxonSet, {
+            translateTaxonNames: this.translateTaxonMap,
+          });
+          // read to first '(';
+          token = await this.skipUntil(/\(/);
+          let buffer = token
+            .split(newickDeliminators)
+            .filter((n) => n.length > 0)
+            .reverse();
+          while (!parser.isDone()) {
+            while (buffer.length > 0) {
+              const c = buffer.pop();
+              notNull(c, `Unexpectedly hit the end of the buffer`);
+              parser.parseCharacter(c);
+            }
+            if (!parser.isDone()) {
+              // get next token
+              token = await this.nextToken();
+              buffer = token
+                .split(newickDeliminators)
+                .filter((n) => n.length > 0)
+                .reverse();
+            }
+          }
+          const tree = parser.getTree();
+          yield tree;
+          break;
+        }
+        case /end/i.test(command):
+          await this.skipSemiColon();
+          this.hasTree = false;
+          // Give up the ghost.
+          //TODO read to the end of the file.
+          this.reader.releaseLock();
+          keepGoing = false;
+          break;
+        default:
+          throw new Error(
+            `Reached impossible code in treeblock block "${command}"`,
+          );
+      }
+    }
+  }
+}

package/src/evo/tree/parsers/stream-reader/nexus-tokenizer.ts

@@ -0,0 +1,77 @@
+import { notNull } from "../../../../utils";
+
+export enum STATUS {
+  PARSING = "parsing",
+  IN_SINGLE_QUOTE = "in single quote",
+  IN_DOUBLE_QUOTE = "in double quote",
+  IN_COMMENT = "in comment",
+}
+
+export function nexusTokenizer() {
+  return {
+    lastChunk: "",
+    status: STATUS.PARSING,
+    end: "",
+    start() {},
+    transform(chunk: string, controller: { enqueue: (arg0: string) => void }) {
+      // not really any but we'll see
+      const data = this.lastChunk + chunk;
+      let buffer = "";
+      for (let i = 0; i < data.length; i++) {
+        const char = data[i];
+        notNull(char, `Internal Error. Hit empty character in array`);
+        if (this.status === STATUS.PARSING) {
+          // on the look out for quotes and comments
+          [this.status, this.end] = getStatusAndEnd(char);
+          if (this.status === STATUS.IN_COMMENT) {
+            // clear the buffer and send the comment separate
+            if (buffer.length > 0) {
+              controller.enqueue(buffer); //pass it on
+            }
+            buffer = "";
+          }
+        } else if (char === this.end) {
+          this.status = STATUS.PARSING;
+          this.end = "";
+        }
+        // if not in quote and hit a space then send it on.
+        if (this.status === STATUS.PARSING && /\s|;|\]/.test(char)) {
+          if (buffer.length > 0) {
+            if (/\]/.test(char)) {
+              buffer += char;
+            } // close the comment and pass
+            controller.enqueue(buffer); //pass it on
+            buffer = "";
+          }
+          if (/;/.test(char)) {
+            controller.enqueue(char); // sent the ';'
+          }
+        } else {
+          buffer += char;
+        }
+      }
+      this.lastChunk = buffer;
+    },
+    flush(controller: { enqueue: (arg0: string) => void }) {
+      if (this.lastChunk) {
+        controller.enqueue(this.lastChunk);
+      }
+    },
+  };
+}
+
+function getStatusAndEnd(char: string): [STATUS, string] {
+  if (char === "'") {
+    return [STATUS.IN_SINGLE_QUOTE, "'"];
+  }
+  if (char === '"') {
+    return [STATUS.IN_DOUBLE_QUOTE, '"'];
+  }
+  if (char === "[") {
+    return [STATUS.IN_COMMENT, "]"];
+  }
+  return [STATUS.PARSING, ""];
+}
+
+export const newickDeliminators =
+  /\s*('[^']+'|"[^"]+"|\[&[^[]+]|,|:|\)|\(|;)\s*/;

package/src/evo/tree/parsers/stream-reader/nexus-transform-stream.txt

@@ -0,0 +1,109 @@
+import { TaxonSet, Tree } from "../..";
+
+enum NexusBlock {
+    Taxa = "taxa",
+    Trees = "trees",
+    Other = "other"
+}
+
+enum STATUS {
+    BLOCK_NAME_NEXT,
+    SKIP_SEMICOLON,
+    PARSING,
+    SKIPPING,
+    IN_BLOCK,
+    SKIP_UNTIL,
+}
+
+export function NexusTransformStream(){
+    // local parameters
+    const taxonSet = new TaxonSet();
+    let currentBlock :NexusBlock | undefined= undefined;
+    let status = STATUS.PARSING;
+    let command = undefined;
+
+    function processChunk(chunk:string):Tree|undefined{
+        if(status === STATUS.SKIP_SEMICOLON){
+            if(!chunk.match(/;$/)) {
+                throw `expected ";" got ${chunk}`;
+            }
+            status = STATUS.PARSING;
+            return;
+        }else if(status === STATUS.SKIPPING){
+            if(chunk.match(/\bend;/i)){
+                status = STATUS.PARSING;
+                currentBlock = undefined;
+                return;
+            }else{
+                return;
+            }
+        } else if(status === STATUS.BLOCK_NAME_NEXT){
+            if(chunk.match(/taxa/i)){
+                currentBlock = NexusBlock.Taxa;
+                status = STATUS.SKIP_SEMICOLON;
+            }else if(chunk.match(/trees/i)){
+                currentBlock = NexusBlock.Trees;
+                status = STATUS.SKIP_SEMICOLON;
+            }else{
+            console.log(
+                    `skipping block ${chunk}. Only parsing blocks are taxa and trees for now.`,
+                  )
+            }
+            status = STATUS.SKIPPING;
+        }else if(chunk.match(/\bbegin/i)){
+            status = STATUS.BLOCK_NAME_NEXT;
+            return;
+        }else if(status === STATUS.PARSING && currentBlock === NexusBlock.Taxa){
+            if(command)
+        }
+       
+
+        return parseTree(chunk);
+        
+    }
+    
+    function parseNextBlock(chunk:string){
+        const blockName = getNextBlockName(chunk);
+        switch (blockName) {
+          case "taxa":
+            this.currentBlock = "taxa"
+            await this.parseTaxaBlock()
+            break
+          case "trees":
+            this.currentBlock = "trees"
+            break
+          default:
+            console.log(
+              `skipping block ${blockName}. Only parsing blocks are taxa and trees for now.`,
+            )
+            await this.readToEndOfBlock()
+        }
+      }
+
+      function getNextBlockName(chunck ) {
+        while (true) {
+          const value = await this.nextToken()
+    
+          if (value.match(/\bbegin/i)) {
+            const token = await this.nextToken()
+            this.skipSemiColon()
+            return token
+          }
+        }
+      }
+
+    return {
+        start(){},
+        transform(chunk: string, controller: { enqueue: (arg0: Tree) => void }) {
+            
+            const tree = processChunk(chunk);
+            
+            if(tree){
+                controller.enqueue(tree);
+            }
+
+        },
+        flush(controller: any) { }
+    }
+
+}

package/src/evo/tree/taxa/helper-functions.ts

@@ -0,0 +1,46 @@
+import type { Maybe, Undefinable } from "@figtreejs/maybe/maybe";
+import { Nothing, Some, MaybeType } from "@figtreejs/maybe/maybe";
+import type { Taxon, TaxonSetData } from "./taxon";
+
+export function maybeGetNameFromIndex(
+  data: TaxonSetData,
+  id: Maybe<number> | number,
+): Maybe<string> {
+  let n: number;
+  if (id instanceof Object) {
+    if (id.type === MaybeType.Nothing) {
+      return id;
+    } else {
+      n = id.value;
+    }
+  } else {
+    n = id;
+  }
+  const name = data.allNames[n] as Undefinable<string>;
+  if (name === undefined) {
+    return Nothing();
+  }
+  return Some(name);
+}
+
+export function maybeGetTaxonByName(
+  data: TaxonSetData,
+  id: Maybe<string> | string,
+): Maybe<Taxon> {
+  let n: string;
+  if (id instanceof Object) {
+    if (id.type === MaybeType.Nothing) {
+      return id;
+    } else {
+      n = id.value;
+    }
+  } else {
+    n = id;
+  }
+  const taxon = data.byName[n] as Undefinable<Taxon>;
+  if (taxon === undefined) {
+    return Nothing();
+  } else {
+    return Some(taxon);
+  }
+}

package/src/evo/tree/taxa/index.ts

@@ -0,0 +1 @@
+export * from "./taxon";