@graphty/graph-io 0.3.46 → 0.3.48

package/src/sources/cypher/transports/query-api.ts

@@ -0,0 +1,100 @@
+/**
+ * The `query-api` transport (design section 12.2): Neo4j's `POST /db/{database}/query/v2`, on by
+ * default since Neo4j 5.26 and on Aura, with no driver. It asks for typed JSON, so integers keep
+ * every digit, and sends `accessMode: "Read"`, which the server enforces. CORS is open by default,
+ * so it works from a page on https.
+ */
+
+import { originOf } from "../../credentials.js";
+import { failOn } from "../../fail.js";
+import { type QueryResult } from "../flatten.js";
+import { type CypherProfile, neo4jProfile } from "../profiles.js";
+import { fromTypedJson } from "../types-map.js";
+import { parseJson, serverError, type Transport, type TransportInput } from "./transport.js";
+
+/** The media type of Neo4j's typed JSON. */
+const TYPED_JSON = "application/vnd.neo4j.query.v1.1";
+
+/** The statuses whose JSON body says what went wrong. */
+const ERROR_STATUSES = [400, 404, 500];
+
+/** The shape of a Query API answer. */
+interface QueryApiBody {
+    readonly data?: { readonly fields: readonly string[]; readonly values: readonly (readonly unknown[])[] };
+    readonly errors?: readonly { readonly code?: string; readonly message?: string }[];
+}
+
+/**
+ * Make the transport.
+ * @param input - the context, params and mapper
+ * @returns the transport
+ */
+export function queryApiTransport(input: TransportInput): Transport {
+    const { ctx, params, m } = input;
+    const database = params.database ?? "neo4j";
+    const path = `db/${encodeURIComponent(database)}/query/v2`;
+    const origin = ctx.endpoint === null ? "" : originOf(ctx.endpoint);
+
+    const run = async (statement: string, parameters: Readonly<Record<string, unknown>>): Promise<QueryResult> => {
+        const response = await ctx.fetch(path, {
+            method: "POST",
+            idempotent: true,
+            allowStatus: ERROR_STATUSES,
+            headers: { "Content-Type": "application/json", Accept: TYPED_JSON },
+            body: JSON.stringify({ statement, parameters, accessMode: "Read" }),
+        });
+        const body = parseJson(await response.text()) as QueryApiBody | null;
+        const first = body?.errors?.[0];
+        if (first !== undefined) {
+            serverError(
+                ctx.report,
+                {
+                    code: first.code ?? null,
+                    message: first.message ?? "",
+                    status: response.status,
+                    showable: response.status >= 400 && response.status < 500,
+                },
+                origin,
+                params.database,
+            );
+        }
+        if (response.status === 404) {
+            // no Query API here: a Neo4j before 5.26 answers 404, and speaks only Bolt
+            failOn(ctx.report, "E_SOURCE_NOT_FOUND", `${origin} has no Query API at ${path}`, {
+                what: "url",
+                value: `${origin}/${path}`,
+            });
+        }
+        if (!response.ok) {
+            failOn(ctx.report, "E_FETCH", `${origin} answered ${String(response.status)}`, {
+                url: `${origin}/${path}`,
+                status: response.status,
+            });
+        }
+        if (body?.data === undefined) {
+            failOn(ctx.report, "E_SOURCE_RESPONSE", "the Query API answer has no data", {
+                expected: "json",
+                contentType: response.headers.get("content-type"),
+            });
+        }
+        const { fields, values } = body.data;
+        return { fields, rows: values.map((row) => row.map((v, i) => fromTypedJson(v, fields[i] ?? "", m))) };
+    };
+
+    return {
+        run,
+        // the Query API exists from Neo4j 5.26, so ids are elementId()
+        profile: () => Promise.resolve(neo4jProfile(null)),
+        probe: async (): Promise<CypherProfile> => {
+            const res = await run(
+                "CALL dbms.components() YIELD name, versions, edition RETURN name, versions, edition",
+                {},
+            );
+            const kernel = res.rows.find((r) => r[0] === "Neo4j Kernel") ?? res.rows[0];
+            const versions = kernel?.[1];
+            const version = Array.isArray(versions) && typeof versions[0] === "string" ? versions[0] : null;
+            return neo4jProfile(version);
+        },
+        close: () => Promise.resolve(),
+    };
+}

package/src/sources/cypher/transports/transport.ts

@@ -0,0 +1,139 @@
+/**
+ * What every Cypher transport is (design section 12.2): something that runs one statement with
+ * its parameters and returns mapped rows, knows the server's profile, and closes. Also the one
+ * place a server's own error code becomes a source issue code.
+ */
+
+import { type ImportReportBuilder } from "../../../common/report.js";
+import { failOn } from "../../fail.js";
+import { type SourceContext } from "../../types.js";
+import { type QueryResult } from "../flatten.js";
+import { type CypherParams } from "../params.js";
+import { type CypherProfile } from "../profiles.js";
+import { type MapContext } from "../types-map.js";
+
+/** One way a Cypher statement travels to a server. Internal. */
+export interface Transport {
+    /**
+     * Run one read-only statement.
+     * @param statement - the statement
+     * @param parameters - its parameters, sent separately
+     * @returns the mapped result
+     */
+    run(statement: string, parameters: Readonly<Record<string, unknown>>): Promise<QueryResult>;
+    /**
+     * The server's profile, detected once.
+     * @returns the profile
+     */
+    profile(): Promise<CypherProfile>;
+    /**
+     * The server facts `probe` reports, which may cost a request.
+     * @returns the profile with its version filled in where the server says it
+     */
+    probe(): Promise<CypherProfile>;
+    /** Release the connection. */
+    close(): Promise<void>;
+}
+
+/** What a transport is made from. */
+export interface TransportInput {
+    /** The load's context. */
+    readonly ctx: SourceContext;
+    /** The params, with defaults. */
+    readonly params: CypherParams;
+    /** The type mapper's context. */
+    readonly m: MapContext;
+}
+
+/** The longest server message a report keeps. */
+const MAX_SERVER_MESSAGE = 500;
+
+/** What a server said about a failed statement. */
+interface ServerError {
+    /** The service's own code, or null. */
+    readonly code: string | null;
+    /** Its message, or "". */
+    readonly message: string;
+    /** The HTTP status, or null over Bolt. */
+    readonly status: number | null;
+    /** Whether the message came from a JSON 4xx body (or a Bolt client error), so a report may keep it. */
+    readonly showable: boolean;
+}
+
+/**
+ * Raise the source issue a server error is: a login failure, a refusal, a server timeout, a
+ * missing database, or E_SOURCE_QUERY with the line and column the message names.
+ * @param report - the load's report
+ * @param err - what the server said
+ * @param origin - the origin that answered
+ * @param database - the database asked for, if any
+ * @returns never; it throws
+ */
+export function serverError(
+    report: ImportReportBuilder,
+    err: ServerError,
+    origin: string,
+    database: string | undefined,
+): never {
+    const code = err.code ?? "";
+    if (/Security\.Unauthorized|AuthenticationRateLimit|UnauthorizedException/i.test(code)) {
+        return failOn(report, "E_SOURCE_AUTH", `${origin} refused the login`, {
+            origin,
+            status: err.status,
+            scheme: null,
+            serverCode: err.code,
+        });
+    }
+    if (/Security\.Forbidden|AccessMode|AccessDenied/i.test(code)) {
+        return failOn(report, "E_SOURCE_FORBIDDEN", `${origin} refused the statement`, {
+            status: err.status,
+            serverCode: err.code,
+        });
+    }
+    if (/TimedOut|Timeout|TimeLimitExceeded/i.test(code)) {
+        return failOn(report, "E_SOURCE_TIMEOUT", `${origin} timed the statement out`, {
+            timeoutMs: null,
+            serverSide: true,
+        });
+    }
+    if (/DatabaseNotFound/i.test(code)) {
+        return failOn(report, "E_SOURCE_NOT_FOUND", `${origin} has no database ${String(database)}`, {
+            what: "database",
+            value: database ?? null,
+        });
+    }
+    const where = /line (\d+), column (\d+)/i.exec(err.message);
+    return failOn(report, "E_SOURCE_QUERY", `${origin} refused the query: ${code}`, {
+        serverCode: err.code,
+        line: where === null ? null : Number(where[1]),
+        column: where === null ? null : Number(where[2]),
+        serverMessage: err.showable && err.message !== "" ? err.message.slice(0, MAX_SERVER_MESSAGE) : null,
+    });
+}
+
+/**
+ * The value of a response body as JSON, or null when it is not JSON.
+ * @param text - the body
+ * @returns the parsed value, or null
+ */
+export function parseJson(text: string): unknown {
+    try {
+        return JSON.parse(text) as unknown;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * The fields and rows of a result given as one object per row (Neptune's `results`).
+ * @param results - the row objects
+ * @param map - maps one value, given its column
+ * @returns the result
+ */
+export function objectRows(
+    results: readonly Readonly<Record<string, unknown>>[],
+    map: (v: unknown, column: string) => unknown,
+): QueryResult {
+    const fields = [...new Set(results.flatMap((r) => Object.keys(r)))];
+    return { fields, rows: results.map((r) => fields.map((f) => map(r[f], f))) };
+}

package/src/sources/cypher/types-map.ts

@@ -0,0 +1,418 @@
+/**
+ * The one type mapper of the Cypher source (design section 10): every transport's wire values --
+ * Neo4j's typed JSON, neo4j-driver's objects, Neptune's plain JSON -- become the same plain values,
+ * and every node, relationship and path becomes a GraphNode, a GraphRel or an array of them.
+ *
+ * - Integers: a number while it is exact; past 2^53 the `long` parameter picks a rounded number
+ *   (W_PRECISION, once per column) or the exact digits as a string.
+ * - Floats, booleans, strings: as they are. Dates and instants: epoch milliseconds. Local times,
+ *   times and durations: their ISO text. Points: `{ srid, x, y, z? }`. Lists and maps: arrays and
+ *   objects of mapped values. Bytes: dropped (W_SOURCE_VALUE).
+ */
+
+import { PRECISION_CODE } from "../../common/codes.js";
+import { type ImportReportBuilder } from "../../common/report.js";
+import { warnOn } from "../fail.js";
+
+/** A node a query returned. */
+export class GraphNode {
+    /**
+     * Create one.
+     * @param id - the server's element id
+     * @param labels - its labels
+     * @param properties - its mapped properties
+     */
+    constructor(
+        readonly id: string,
+        readonly labels: readonly string[],
+        readonly properties: Readonly<Record<string, unknown>>,
+    ) {}
+}
+
+/** A relationship a query returned. */
+export class GraphRel {
+    /**
+     * Create one.
+     * @param id - the server's element id
+     * @param start - the start node's element id
+     * @param end - the end node's element id
+     * @param type - its type
+     * @param properties - its mapped properties
+     */
+    constructor(
+        readonly id: string,
+        readonly start: string,
+        readonly end: string,
+        readonly type: string,
+        readonly properties: Readonly<Record<string, unknown>>,
+    ) {}
+}
+
+/** What the mapper needs: the `long` choice and the report its warnings go to. */
+export interface MapContext {
+    /** Integers past 2^53: "f64" (rounded, W_PRECISION) or "string". */
+    readonly long: "f64" | "string";
+    /** The load's report. */
+    readonly report: ImportReportBuilder;
+}
+
+/** Values past this need a 64-bit integer. */
+const SAFE = BigInt(Number.MAX_SAFE_INTEGER);
+
+/**
+ * An integer from its exact decimal digits.
+ * @param digits - the digits
+ * @param column - the column it belongs to, for the warning
+ * @param m - the map context
+ * @returns a number, or a string past 2^53 under `long: "string"`
+ */
+function integer(digits: string, column: string, m: MapContext): number | string {
+    let big: bigint;
+    try {
+        big = BigInt(digits);
+    } catch {
+        return Number(digits);
+    }
+    if (big <= SAFE && big >= -SAFE) {
+        return Number(big);
+    }
+    if (m.long === "string") {
+        return big.toString();
+    }
+    m.report.warnOnce(
+        "precision",
+        PRECISION_CODE,
+        `column ${column}: an integer beyond 2^53 was stored as the nearest 64-bit float; pass long: "string" to keep every digit`,
+        undefined,
+        `${PRECISION_CODE}:${column}`,
+    );
+    return Number(big);
+}
+
+/**
+ * Epoch milliseconds of an ISO instant, the zone name in brackets ignored; the text itself when it
+ * does not parse.
+ * @param text - the ISO text
+ * @param type - the server's type name
+ * @param column - the column
+ * @param m - the map context
+ * @returns milliseconds, or the text
+ */
+function epoch(text: string, type: string, column: string, m: MapContext): number | string {
+    const ms = Date.parse(withoutZoneName(text));
+    if (Number.isNaN(ms)) {
+        warnOn(m.report, "W_SOURCE_VALUE", `column ${column}: a ${type} kept as text`, { type, column, as: "string" });
+        return text;
+    }
+    return ms;
+}
+
+/**
+ * Warn that a value with no dtype was dropped; the caller leaves the property out.
+ * @param type - the server's type name
+ * @param column - the column
+ * @param m - the map context
+ */
+function dropped(type: string, column: string, m: MapContext): void {
+    warnOn(m.report, "W_SOURCE_VALUE", `column ${column}: a ${type} value was dropped`, {
+        type,
+        column,
+        as: "dropped",
+    });
+}
+
+/**
+ * Map every property of a map, leaving out dropped values.
+ * @param value - the map
+ * @param each - maps one value, given its key
+ * @returns the mapped map
+ */
+function mapEntries(
+    value: Readonly<Record<string, unknown>>,
+    each: (v: unknown, key: string) => unknown,
+): Record<string, unknown> {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value)) {
+        const mapped = each(v, k);
+        if (mapped !== undefined) {
+            out[k] = mapped;
+        }
+    }
+    return out;
+}
+
+/**
+ * A WKT point as Neo4j's typed JSON spells it ("SRID=7203;POINT (1.0 2.0)").
+ * @param text - the text
+ * @returns `{ srid, x, y, z? }`, or the text when it does not parse
+ */
+function wktPoint(text: string): unknown {
+    const match = /^SRID=(\d+);POINT(?: Z)? ?\(([^)]*)\)$/i.exec(text.trim());
+    if (match === null) {
+        return text;
+    }
+    const [x = 0, y = 0, z] = match[2].trim().split(/\s+/).map(Number);
+    return z === undefined ? { srid: Number(match[1]), x, y } : { srid: Number(match[1]), x, y, z };
+}
+
+/** One typed JSON value: `{ "$type": "Integer", "_value": "42" }`. */
+interface Typed {
+    readonly $type: string;
+    readonly _value: unknown;
+}
+
+/** The shape of a typed JSON node's value. */
+interface TypedNode {
+    readonly _element_id: string;
+    readonly _labels: readonly string[];
+    readonly _properties: Readonly<Record<string, unknown>>;
+}
+
+/** The shape of a typed JSON relationship's value. */
+interface TypedRel {
+    readonly _element_id: string;
+    readonly _start_node_element_id: string;
+    readonly _end_node_element_id: string;
+    readonly _type: string;
+    readonly _properties: Readonly<Record<string, unknown>>;
+}
+
+/**
+ * Map a value of Neo4j's typed JSON (the Query API's `application/vnd.neo4j.query.v1.1`).
+ * @param v - the wire value
+ * @param column - the result column or property key it belongs to
+ * @param m - the map context
+ * @returns the plain value, a GraphNode, a GraphRel or an array; undefined when dropped
+ */
+export function fromTypedJson(v: unknown, column: string, m: MapContext): unknown {
+    if (v === null || typeof v !== "object" || !("$type" in v)) {
+        return v ?? null;
+    }
+    const { $type: type, _value: value } = v as Typed;
+    switch (type) {
+        case "Null":
+            return null;
+        case "Boolean":
+        case "String":
+            return value;
+        case "Integer":
+            return integer(String(value), column, m);
+        case "Float":
+            return Number(value);
+        case "Date":
+            return epoch(`${String(value)}T00:00:00Z`, type, column, m);
+        case "OffsetDateTime":
+        case "ZonedDateTime":
+            return epoch(String(value), type, column, m);
+        case "LocalDateTime":
+        case "LocalTime":
+        case "Time":
+        case "Duration":
+            return String(value);
+        case "Point":
+            return wktPoint(String(value));
+        case "Base64":
+            dropped(type, column, m);
+            return undefined;
+        case "List":
+            return (value as readonly unknown[]).map((x) => fromTypedJson(x, column, m));
+        case "Map":
+            return mapEntries(value as Record<string, unknown>, (x) => fromTypedJson(x, column, m));
+        case "Node": {
+            const n = value as TypedNode;
+            return new GraphNode(
+                n._element_id,
+                n._labels,
+                mapEntries(n._properties, (x, k) => fromTypedJson(x, k, m)),
+            );
+        }
+        case "Relationship": {
+            const r = value as TypedRel;
+            return new GraphRel(
+                r._element_id,
+                r._start_node_element_id,
+                r._end_node_element_id,
+                r._type,
+                mapEntries(r._properties, (x, k) => fromTypedJson(x, k, m)),
+            );
+        }
+        case "Path":
+            return (value as readonly unknown[]).map((x) => fromTypedJson(x, column, m));
+        default:
+            warnOn(m.report, "W_SOURCE_VALUE", `column ${column}: a ${type} kept as text`, {
+                type,
+                column,
+                as: "string",
+            });
+            return typeof value === "string" ? value : JSON.stringify(value);
+    }
+}
+
+/** The parts of neo4j-driver the mapper uses: its type checks. */
+export interface DriverTypes {
+    isInt(v: unknown): boolean;
+    isNode(v: unknown): boolean;
+    isRelationship(v: unknown): boolean;
+    isPath(v: unknown): boolean;
+    isPoint(v: unknown): boolean;
+    isDate(v: unknown): boolean;
+    isDateTime(v: unknown): boolean;
+    isLocalDateTime(v: unknown): boolean;
+    isLocalTime(v: unknown): boolean;
+    isTime(v: unknown): boolean;
+    isDuration(v: unknown): boolean;
+}
+
+/** A driver node or relationship, as far as the mapper reads it. */
+interface DriverEntity {
+    readonly identity: unknown;
+    readonly elementId?: string;
+    readonly labels?: readonly string[];
+    readonly type?: string;
+    readonly start?: unknown;
+    readonly end?: unknown;
+    readonly startNodeElementId?: string;
+    readonly endNodeElementId?: string;
+    readonly properties: Readonly<Record<string, unknown>>;
+}
+
+/** A driver path. */
+interface DriverPath {
+    readonly start: unknown;
+    readonly segments: readonly { readonly relationship: unknown; readonly end: unknown }[];
+}
+
+/** A driver point. */
+interface DriverPoint {
+    readonly srid: unknown;
+    readonly x: number;
+    readonly y: number;
+    readonly z?: number;
+}
+
+/**
+ * The text without a trailing bracketed zone name (`...+01:00[Europe/Paris]`).
+ * @param text - the ISO text
+ * @returns the text without the zone name
+ */
+function withoutZoneName(text: string): string {
+    const end = text.length - 1;
+    if (text[end] !== "]") {
+        return text;
+    }
+    const open = text.indexOf("[", text.lastIndexOf("]", end - 1) + 1);
+    return open === -1 || open >= end ? text : text.slice(0, open);
+}
+
+/**
+ * Map a driver node or relationship.
+ * @param e - the driver entity
+ * @param node - whether it is a node
+ * @param m - the map context
+ * @param t - the driver's type checks
+ * @returns a GraphNode or a GraphRel
+ */
+function fromEntity(e: DriverEntity, node: boolean, m: MapContext, t: DriverTypes): GraphNode | GraphRel {
+    const props = mapEntries(e.properties, (x, k) => fromDriver(x, k, m, t));
+    const id = e.elementId ?? String(e.identity);
+    return node
+        ? new GraphNode(id, [...(e.labels ?? [])], props)
+        : new GraphRel(
+              id,
+              e.startNodeElementId ?? String(e.start),
+              e.endNodeElementId ?? String(e.end),
+              e.type ?? "",
+              props,
+          );
+}
+
+/**
+ * Map a driver point.
+ * @param p - the point
+ * @returns srid, x, y and z when it has one
+ */
+function fromPoint(p: DriverPoint): { srid: number; x: number; y: number; z?: number } {
+    const srid = Number(String(p.srid));
+    return p.z === undefined ? { srid, x: p.x, y: p.y } : { srid, x: p.x, y: p.y, z: p.z };
+}
+
+/**
+ * Map a value neo4j-driver returned. Its lossless Integer keeps every digit, so `long` decides.
+ * @param v - the driver value
+ * @param column - the result column or property key
+ * @param m - the map context
+ * @param t - the driver's type checks
+ * @returns the plain value, a GraphNode, a GraphRel or an array; undefined when dropped
+ */
+export function fromDriver(v: unknown, column: string, m: MapContext, t: DriverTypes): unknown {
+    if (v === null || v === undefined) {
+        return null;
+    }
+    if (typeof v !== "object") {
+        return typeof v === "bigint" ? integer(v.toString(), column, m) : v;
+    }
+    // driver values (Integer, the temporal types) spell themselves exactly through toString()
+    const text = (): string => (v as { toString(): string }).toString();
+    if (t.isInt(v)) {
+        return integer(text(), column, m);
+    }
+    if (t.isNode(v) || t.isRelationship(v)) {
+        return fromEntity(v as DriverEntity, t.isNode(v), m, t);
+    }
+    if (t.isPath(v)) {
+        const p = v as DriverPath;
+        return [p.start, ...p.segments.flatMap((s) => [s.relationship, s.end])].map((x) => fromDriver(x, column, m, t));
+    }
+    if (t.isDate(v)) {
+        return epoch(`${text()}T00:00:00Z`, "Date", column, m);
+    }
+    if (t.isDateTime(v)) {
+        return epoch(text(), "DateTime", column, m);
+    }
+    if (t.isLocalDateTime(v) || t.isLocalTime(v) || t.isTime(v) || t.isDuration(v)) {
+        return text();
+    }
+    if (t.isPoint(v)) {
+        return fromPoint(v as DriverPoint);
+    }
+    if (v instanceof Uint8Array || v instanceof ArrayBuffer) {
+        dropped("ByteArray", column, m);
+        return undefined;
+    }
+    if (Array.isArray(v)) {
+        return v.map((x: unknown) => fromDriver(x, column, m, t));
+    }
+    return mapEntries(v as Record<string, unknown>, (x) => fromDriver(x, column, m, t));
+}
+
+/**
+ * Map a value of Neptune's openCypher JSON: nodes `{ "~id", "~entityType": "node", "~labels",
+ * "~properties" }`, relationships adding `~start`, `~end`, `~type`. Its numbers are plain JSON, so
+ * an integer past 2^53 was already rounded on the wire (W_PRECISION).
+ * @param v - the wire value
+ * @param column - the result column or property key
+ * @param m - the map context
+ * @returns the plain value, a GraphNode, a GraphRel or an array
+ */
+export function fromNeptune(v: unknown, column: string, m: MapContext): unknown {
+    if (typeof v === "number" && Number.isInteger(v) && !Number.isSafeInteger(v)) {
+        return integer(BigInt(v).toString(), column, m);
+    }
+    if (v === null || typeof v !== "object") {
+        return v ?? null;
+    }
+    if (Array.isArray(v)) {
+        return v.map((x: unknown) => fromNeptune(x, column, m));
+    }
+    const o = v as Record<string, unknown>;
+    const props = (): Record<string, unknown> =>
+        mapEntries((o["~properties"] ?? {}) as Record<string, unknown>, (x, k) => fromNeptune(x, k, m));
+    switch (o["~entityType"]) {
+        case "node":
+            return new GraphNode(String(o["~id"]), (o["~labels"] ?? []) as string[], props());
+        case "relationship":
+            return new GraphRel(String(o["~id"]), String(o["~start"]), String(o["~end"]), String(o["~type"]), props());
+        default:
+            return mapEntries(o, (x) => fromNeptune(x, column, m));
+    }
+}