@loaders.gl/graphs 4.5.3 → 5.0.0-alpha.10

package/src/lib/build-graph-tables.ts

@@ -0,0 +1,208 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  Bool,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  Utf8,
+  Struct,
+  List,
+  Field,
+  Table,
+  vectorFromArray
+} from 'apache-arrow';
+import type {DataType, Vector} from 'apache-arrow';
+import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+import type {ArrowTable, ObjectRowTable} from '@loaders.gl/schema';
+import type {
+  GraphNode,
+  GraphEdge,
+  GraphOutput,
+  GraphShape,
+  GraphAttributeSchemas,
+  GraphAttributeType
+} from '../graph-types';
+
+/** Constructs named tables without materializing an intermediate plain graph. */
+export function buildGraphTables(
+  nodes: Iterable<GraphNode>,
+  edges: Iterable<GraphEdge>,
+  shape: GraphShape,
+  declarations?: GraphAttributeSchemas
+): GraphOutput {
+  if (shape === 'plain-graph-data') {
+    return {shape, nodes: Array.from(nodes), edges: Array.from(edges)};
+  }
+  if (shape !== 'arrow-table' && shape !== 'object-row-table') {
+    throw new Error(`Unsupported graph shape: ${shape}`);
+  }
+  const nodeTable = buildGraphTable(nodes, false, shape, declarations?.nodes);
+  const edgeTable = buildGraphTable(edges, true, shape, declarations?.edges);
+  // Both tables always have the same representation.
+  return {
+    shape: 'tables',
+    tables: [
+      {name: 'nodes', table: nodeTable},
+      {name: 'edges', table: edgeTable}
+    ]
+  } as GraphOutput;
+}
+
+/** Builds structural columns and a collision-free application attribute struct. */
+function buildGraphTable(
+  records: Iterable<GraphNode | GraphEdge>,
+  isEdge: boolean,
+  shape: 'arrow-table' | 'object-row-table',
+  declarations?: Map<string, GraphAttributeType>
+): ArrowTable | ObjectRowTable {
+  const columnNames = isEdge
+    ? ['id', 'sourceId', 'targetId', 'directed', 'label', 'attributes']
+    : ['id', 'label', 'attributes'];
+  const columns = new Map<string, unknown[]>(columnNames.map(name => [name, []]));
+  for (const record of records) {
+    for (const name of columnNames) {
+      const value = (record as unknown as Record<string, unknown>)[name];
+      columns
+        .get(name)!
+        .push(
+          name === 'id' || name === 'sourceId' || name === 'targetId'
+            ? String(value)
+            : (value ?? null)
+        );
+    }
+  }
+  if (shape === 'object-row-table') {
+    return {
+      shape,
+      data: columns
+        .get('id')!
+        .map((_, index) =>
+          Object.fromEntries(columnNames.map(name => [name, columns.get(name)![index]]))
+        )
+    };
+  }
+  const vectors: Record<string, Vector> = Object.create(null);
+  for (const [name, values] of columns) {
+    const type =
+      name === 'attributes'
+        ? inferAttributeType(values, declarations)
+        : name === 'directed'
+          ? new Bool()
+          : new Utf8();
+    vectors[name] = vectorFromArray(
+      values.map(value => normalizeValue(value, type)),
+      type
+    );
+  }
+  const data = new Table(vectors);
+  return {shape, schema: convertArrowToSchema(data.schema), data};
+}
+
+/** Infers a struct using the union of declared and observed application keys. */
+function inferAttributeType(
+  values: unknown[],
+  declarations?: Map<string, GraphAttributeType>
+): Struct {
+  const names = new Set(declarations?.keys());
+  for (const value of values) {
+    if (value && typeof value === 'object') {
+      for (const name of Object.keys(value)) {
+        names.add(name);
+      }
+    }
+  }
+  return new Struct(
+    Array.from(names, name => {
+      const fieldValues = values.map(
+        value => (value as Record<string, unknown> | null)?.[name] ?? null
+      );
+      return new Field(name, inferValueType(fieldValues, declarations?.get(name)), true);
+    })
+  );
+}
+
+/** Preserves declared scalar types where valid and infers nested DOT attributes. */
+function inferValueType(values: unknown[], declaration?: GraphAttributeType): DataType {
+  const present = values.filter(value => value !== null && value !== undefined);
+  if (declaration && typeof declaration === 'object') {
+    if (!present.every(value => Array.isArray(value))) return new Utf8();
+    return new List(
+      new Field(
+        'item',
+        inferValueType(
+          present.flatMap(value => value as unknown[]),
+          declaration.list
+        ),
+        true
+      )
+    );
+  }
+  if (declaration) {
+    const declaredTypes = {
+      boolean: new Bool(),
+      int: new Int32(),
+      long: new Int64(),
+      float: new Float32(),
+      double: new Float64(),
+      string: new Utf8()
+    };
+    const valid = present.every(value => {
+      switch (declaration) {
+        case 'boolean':
+          return typeof value === 'boolean';
+        case 'int':
+          return (
+            typeof value === 'number' &&
+            Number.isInteger(value) &&
+            value >= -2147483648 &&
+            value <= 2147483647
+          );
+        case 'long':
+          return typeof value === 'bigint' && value >= -(2n ** 63n) && value < 2n ** 63n;
+        case 'float':
+        case 'double':
+          return typeof value === 'number';
+        case 'string':
+          return typeof value === 'string';
+      }
+    });
+    return valid ? declaredTypes[declaration] : new Utf8();
+  }
+  if (present.length && present.every(value => typeof value === 'boolean')) return new Bool();
+  if (present.length && present.every(value => typeof value === 'number')) return new Float64();
+  if (present.length && present.every(value => Array.isArray(value))) {
+    return new List(
+      new Field('item', inferValueType(present.flatMap(value => value as unknown[])), true)
+    );
+  }
+  if (
+    present.length &&
+    present.every(value => typeof value === 'object' && !Array.isArray(value))
+  ) {
+    return inferAttributeType(present);
+  }
+  return new Utf8();
+}
+
+/** Normalizes missing fields and mixed values for the selected Arrow type. */
+function normalizeValue(value: unknown, type: DataType): unknown {
+  if (value === null || value === undefined) return null;
+  if (type instanceof Struct) {
+    return Object.fromEntries(
+      type.children.map(field => [
+        field.name,
+        normalizeValue((value as Record<string, unknown>)[field.name], field.type)
+      ])
+    );
+  }
+  if (type instanceof List)
+    return (value as unknown[]).map(item => normalizeValue(item, type.valueType));
+  if (type instanceof Utf8 && typeof value !== 'string') {
+    return typeof value === 'object' ? JSON.stringify(value) : String(value);
+  }
+  return value;
+}

package/src/lib/parse-dot.ts

@@ -2,7 +2,9 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {DOTGraphData} from '../dot-types';
+import type {DOTGraphData, DOTOutput} from '../dot-types';
+import type {GraphShape, GraphNode, GraphEdge} from '../graph-types';
+import {buildGraphTables} from './build-graph-tables';
 
 /** Internal DOT parsing representation. */
 type DOTAttributeMap = Record<string, unknown>;
@@ -62,25 +64,45 @@ type DOTParseResult = {
 };
 
 /** Parses DOT source into plain graph records without a visualization dependency. */
-export function parseDOT(input: string): DOTGraphData {
+export function parseDOT(input: string): DOTGraphData;
+export function parseDOT(input: string, shape: GraphShape): DOTOutput;
+export function parseDOT(input: string, shape: GraphShape = 'plain-graph-data'): DOTOutput {
   const parsed = parseDOTDocument(input);
-  const subgraphs = Array.from(parsed.subgraphs.values(), (subgraph) => ({
-    ...subgraph,
-    attributes: {...subgraph.attributes}
-  }));
   return {
-    shape: 'plain-graph-data',
-    nodes: Array.from(parsed.nodes.values(), (node) => ({
+    ...buildGraphTables(iterateNodes(parsed), iterateEdges(parsed), shape),
+    metadata: {
+      id: parsed.id,
+      directed: parsed.directed,
+      strict: parsed.strict,
+      attributes: {...parsed.graphAttributes},
+      subgraphs: Array.from(parsed.subgraphs.values(), subgraph => ({
+        ...subgraph,
+        attributes: {...subgraph.attributes}
+      }))
+    }
+  };
+}
+
+/** Emits finalized nodes without creating an intermediate record array. */
+function* iterateNodes(parsed: DOTParseResult): Iterable<GraphNode> {
+  for (const node of parsed.nodes.values()) {
+    yield {
       id: node.id,
       ...(typeof node.attributes.label === 'string' ? {label: node.attributes.label} : {}),
       attributes: {
         ...node.attributes,
         ...(node.subgraphs.size
-          ? {subgraphs: Array.from(node.subgraphs, (id) => describeSubgraph(id, parsed.subgraphs))}
+          ? {subgraphs: Array.from(node.subgraphs, id => describeSubgraph(id, parsed.subgraphs))}
           : {})
       }
-    })),
-    edges: parsed.edges.map((edge) => ({
+    };
+  }
+}
+
+/** Emits coalesced edges in document order. */
+function* iterateEdges(parsed: DOTParseResult): Iterable<GraphEdge> {
+  for (const edge of parsed.edges) {
+    yield {
       id: edge.id,
       sourceId: edge.sourceId,
       targetId: edge.targetId,
@@ -89,18 +111,11 @@ export function parseDOT(input: string): DOTGraphData {
       attributes: {
         ...edge.attributes,
         ...(edge.subgraphs.length
-          ? {subgraphs: edge.subgraphs.map((id) => describeSubgraph(id, parsed.subgraphs))}
+          ? {subgraphs: edge.subgraphs.map(id => describeSubgraph(id, parsed.subgraphs))}
           : {})
       }
-    })),
-    metadata: {
-      id: parsed.id,
-      directed: parsed.directed,
-      strict: parsed.strict,
-      attributes: {...parsed.graphAttributes},
-      subgraphs
-    }
-  };
+    };
+  }
 }
 
 /** Copies a subgraph descriptor into node or edge attributes. */
@@ -393,6 +408,10 @@ class DOTParser {
       const existingEdge = this.result.strict ? this.strictEdges.get(strictKey) : undefined;
       if (existingEdge) {
         existingEdge.attributes = {...existingEdge.attributes, ...explicitAttributes};
+        const explicitId = explicitAttributes.id ?? explicitAttributes.Id ?? explicitAttributes.ID;
+        if (typeof explicitId === 'string' || typeof explicitId === 'number') {
+          existingEdge.id = String(explicitId);
+        }
         existingEdge.directed = deriveDirectedFlag(existingEdge.attributes, directed);
         existingEdge.subgraphs = Array.from(new Set([...existingEdge.subgraphs, ...membership]));
         continue;
@@ -496,7 +515,7 @@ class DOTParser {
       node = {id, attributes: {...defaults}, subgraphs: new Set()};
       this.result.nodes.set(id, node);
     }
-    membership.forEach((subgraphId) => node.subgraphs.add(subgraphId));
+    membership.forEach(subgraphId => node.subgraphs.add(subgraphId));
     return node;
   }
 
@@ -742,13 +761,17 @@ function readQuotedString(input: string, startIndex: number): {value: string; ne
   throw new Error('Unterminated string literal in DOT source.');
 }
 
-/** Decodes one quoted-string escape. */
+/** Decodes one quoted-string escape or removes a physical line continuation. */
 function readEscapedCharacter(
   input: string,
   startIndex: number
 ): {value: string; nextIndex: number} {
   const next = input[startIndex];
   switch (next) {
+    case '\n':
+      return {value: '', nextIndex: startIndex + 1};
+    case '\r':
+      return {value: '', nextIndex: startIndex + (input[startIndex + 1] === '\n' ? 2 : 1)};
     case 'n':
     case 'l':
     case 'L':
@@ -765,7 +788,7 @@ function readEscapedCharacter(
       if (typeof next === 'undefined') {
         throw new Error('Unterminated escape sequence in DOT source.');
       }
-      return {value: next, nextIndex: startIndex + 1};
+      return {value: `\\${next}`, nextIndex: startIndex + 1};
     }
   }
 }