@graphty/graph-io 0.3.46 → 0.3.48

package/src/sources/define.ts

@@ -0,0 +1,195 @@
+/**
+ * `defineSource` and the registration checks (design section 2): the id form, exactly one read
+ * method, and the declared features against the methods present. Also the parameter checks every
+ * load runs against the descriptor before any request.
+ */
+
+import { type ImportReportBuilder } from "../common/report.js";
+import { failAlone, failOn } from "./fail.js";
+import {
+    type GraphSource,
+    type LazySource,
+    type SourceDescriptor,
+    type SourceFeature,
+    type SourceParamDescriptor,
+} from "./types.js";
+
+/** Lowercase letters, digits and single dashes between them. */
+const ID_FORM = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+
+/** The optional methods and the feature each one is. */
+const FEATURE_METHODS: readonly (readonly [SourceFeature, "expand" | "probe" | "list" | "validate"])[] = [
+    ["expand", "expand"],
+    ["probe", "probe"],
+    ["list", "list"],
+    ["validate", "validate"],
+];
+
+/**
+ * Refuse a malformed source.
+ * @param id - the source id
+ * @param reason - why
+ * @param message - for logs
+ */
+function invalid(id: string, reason: string, message: string): never {
+    failAlone("E_SOURCE_INVALID", message, { id, reason });
+}
+
+/**
+ * Check a source id.
+ * @param id - the id
+ */
+function checkSourceId(id: unknown): asserts id is string {
+    if (typeof id !== "string" || !ID_FORM.test(id)) {
+        invalid(String(id), "id", `source id ${JSON.stringify(id)} must be lowercase letters, digits and dashes`);
+    }
+}
+
+/**
+ * The features a loaded source has, from its methods.
+ * @param source - the source
+ * @returns the features
+ */
+function featuresOf(source: GraphSource<never>): readonly SourceFeature[] {
+    return FEATURE_METHODS.filter(([, method]) => typeof source[method] === "function").map(([f]) => f);
+}
+
+/**
+ * Check a loaded source: its id, exactly one read method, and its declared features.
+ * @param source - the source
+ * @param declared - features a lazy registration declared, if any
+ */
+export function checkSource(source: GraphSource<never>, declared?: SourceDescriptor): void {
+    checkSourceId((source as { id?: unknown }).id);
+    const reads = (["records", "file", "import"] as const).filter((m) => typeof source[m] === "function");
+    if (reads.length !== 1) {
+        invalid(source.id, "read-methods", `source ${source.id} must have exactly one of records, file and import`);
+    }
+    const features = (declared ?? source.descriptor)?.features;
+    if (features !== undefined) {
+        const has = new Set(featuresOf(source));
+        for (const f of features) {
+            if (f !== "page" && !has.has(f)) {
+                invalid(source.id, "features", `source ${source.id} declares ${f} but has no ${f} method`);
+            }
+        }
+    }
+}
+
+/**
+ * Check a lazy registration before its code is loaded.
+ * @param source - the lazy source
+ */
+export function checkLazySource(source: LazySource): void {
+    checkSourceId(source.id);
+    if (typeof source.load !== "function" || typeof source.descriptor !== "object") {
+        invalid(source.id, "lazy", `lazy source ${source.id} needs a descriptor and a load function`);
+    }
+}
+
+/**
+ * Define a network source: returns it unchanged, with its parameter type inferred, after the
+ * registration checks (an id of lowercase letters, digits and dashes; exactly one of `records`,
+ * `file` and `import`; declared features backed by methods). A malformed source throws an
+ * ImportError whose `issue.code` is E_SOURCE_INVALID. Experimental.
+ * @param source - the source
+ * @returns the same source
+ * @example
+ * ```ts
+ * const people = defineSource({
+ *     id: "people",
+ *     async records(params: { team: string }, ctx) {
+ *         const rows = await ctx.fetchJson<{ id: string; name: string }[]>(`/api/people?team=${params.team}`);
+ *         return { nodes: rows };
+ *     },
+ * });
+ * ```
+ * @category Network sources
+ */
+export function defineSource<P>(source: GraphSource<P>): GraphSource<P> {
+    checkSource(source);
+    return source;
+}
+
+/**
+ * Why a number or integer param is wrong, if it is.
+ * @param p - the param's descriptor
+ * @param value - the value given
+ * @returns the reason, or null when it is fine
+ */
+function numberProblem(p: SourceParamDescriptor, value: unknown): string | null {
+    if (typeof value !== "number" || Number.isNaN(value) || (p.type === "integer" && !Number.isInteger(value))) {
+        return "type";
+    }
+    if ((p.min !== undefined && value < p.min) || (p.max !== undefined && value > p.max)) {
+        return "range";
+    }
+    return null;
+}
+
+/**
+ * Why a URL param is wrong, if it is.
+ * @param value - the value given
+ * @returns the reason, or null when it is fine
+ */
+function urlProblem(value: unknown): string | null {
+    let url: URL;
+    try {
+        url = new URL(typeof value === "string" ? value : "");
+    } catch {
+        return "type";
+    }
+    return url.username !== "" || url.password !== "" ? "credentials-in-url" : null;
+}
+
+/**
+ * Why a present param value is wrong for its descriptor, if it is.
+ * @param p - the param's descriptor
+ * @param value - the value given (not undefined or null)
+ * @returns the reason, or null when it is fine
+ */
+function paramProblem(p: SourceParamDescriptor, value: unknown): string | null {
+    switch (p.type) {
+        case "integer":
+        case "number":
+            return numberProblem(p, value);
+        case "boolean":
+            return typeof value === "boolean" ? null : "type";
+        case "enum":
+            return p.values !== undefined && !p.values.some((v) => v.value === value) ? "enum" : null;
+        case "url":
+            return urlProblem(value);
+        default:
+            return null;
+    }
+}
+
+/**
+ * Check params against a descriptor before any request: required ones present, numbers and
+ * integers in range, booleans, enums, URLs (never with a user name or password in them).
+ * @param params - the caller's params
+ * @param descriptor - the source's descriptor
+ * @param report - the load's report
+ */
+export function checkParams(
+    params: Readonly<Record<string, unknown>>,
+    descriptor: SourceDescriptor | undefined,
+    report: ImportReportBuilder,
+): void {
+    if (typeof params !== "object" || (params as unknown) === null) {
+        failOn(report, "E_SOURCE_PARAM", "params must be an object", { name: "params", reason: "type" });
+    }
+    for (const p of descriptor?.params ?? []) {
+        const value = params[p.name];
+        const missing = value === undefined || value === null;
+        let reason: string | null = null;
+        if (!missing) {
+            reason = paramProblem(p, value);
+        } else if (p.required === true && p.default === undefined) {
+            reason = "required";
+        }
+        if (reason !== null) {
+            failOn(report, "E_SOURCE_PARAM", `parameter ${p.name}: ${reason}`, { name: p.name, reason });
+        }
+    }
+}

package/src/sources/fail.ts

@@ -0,0 +1,70 @@
+/**
+ * Raising a network source issue: one place that turns a code and its params into an ImportError
+ * whose `issue.params` and `details` both carry the params, so callers branch on
+ * `err.issue.code` and read `err.issue.params` as with every other graph-io failure.
+ */
+
+import { ImportReportBuilder } from "../common/report.js";
+import { type IssueCategory } from "../types.js";
+import { type SourceIssueCode } from "./codes.js";
+
+/** The category each code is filed under. */
+const CATEGORY: Readonly<Partial<Record<SourceIssueCode, IssueCategory>>> = {
+    E_SOURCE_PARAM: "validation-error",
+    E_SOURCE_INVALID: "validation-error",
+    E_SOURCE_RESPONSE: "parse-error",
+    E_SOURCE_NOT_A_GRAPH: "parse-error",
+    E_SOURCE_NOT_FOUND: "missing-value",
+    E_SOURCE_UNKNOWN: "missing-value",
+};
+
+/**
+ * Abort a load with a source error, carrying the load's report so far plus the error.
+ * @param report - the load's report
+ * @param code - the code
+ * @param message - a plain-ASCII message, for logs
+ * @param params - the issue's values
+ * @returns never; it throws the ImportError
+ */
+export function failOn(
+    report: ImportReportBuilder,
+    code: SourceIssueCode,
+    message: string,
+    params: Readonly<Record<string, unknown>>,
+): never {
+    // a fork, so a failure the runner turns into a warning (a later page's 401, a cap) leaves no
+    // error in the load's own report; the ImportError carries everything recorded so far
+    return report.fork().failWith(CATEGORY[code] ?? "unsupported", code, message, undefined, params, params);
+}
+
+/**
+ * Raise a source error outside a load (registration, a credential factory).
+ * @param code - the code
+ * @param message - a plain-ASCII message, for logs
+ * @param params - the issue's values
+ * @param format - the report's format name
+ */
+export function failAlone(
+    code: SourceIssueCode,
+    message: string,
+    params: Readonly<Record<string, unknown>>,
+    format = "source",
+): never {
+    failOn(new ImportReportBuilder(format, 0), code, message, params);
+}
+
+/**
+ * Record a source warning on a report.
+ * @param report - the report
+ * @param code - the code
+ * @param message - a plain-ASCII message, for logs
+ * @param params - the issue's values
+ */
+export function warnOn(
+    report: ImportReportBuilder,
+    code: SourceIssueCode,
+    message: string,
+    params: Readonly<Record<string, unknown>>,
+): void {
+    report.warning("unsupported", code, message, undefined, params);
+}

package/src/sources/ids.ts

@@ -0,0 +1,74 @@
+/**
+ * The public names of network sources: one constant per built-in source id, and the reserved
+ * `_source.` attribute names. Nothing else in graph-io spells them, so a rename is one edit here.
+ *
+ * Experimental: the ids and the `_source.` namespace await the owner's final word (design
+ * section 22, items 4 and 5).
+ */
+
+/**
+ * The NDEx source (www.ndexbio.org). Experimental.
+ * @category Network sources
+ */
+export const NDEX_SOURCE_ID = "ndex";
+
+/**
+ * The generic Cypher source, for any openCypher server. Experimental.
+ * @category Network sources
+ */
+export const CYPHER_SOURCE_ID = "cypher";
+
+/**
+ * The Neo4j preset of the Cypher source. Experimental.
+ * @category Network sources
+ */
+export const NEO4J_SOURCE_ID = "neo4j";
+
+/**
+ * The Amazon Neptune preset of the Cypher source. Experimental.
+ * @category Network sources
+ */
+export const NEPTUNE_SOURCE_ID = "neptune";
+
+/**
+ * The generic REST mapping source. Experimental.
+ * @category Network sources
+ */
+export const REST_SOURCE_ID = "rest";
+
+/**
+ * The OpenAlex preset of the REST mapping source. Experimental.
+ * @category Network sources
+ */
+export const OPENALEX_SOURCE_ID = "openalex";
+
+/**
+ * The prefix of the reserved attribute namespace sources and graphty-element write facts under.
+ * @category Network sources
+ */
+export const SOURCE_ATTR_PREFIX = "_source.";
+
+/**
+ * Node attribute: true on a stub, a node known only by its id until it is hydrated or expanded.
+ * @category Network sources
+ */
+export const SOURCE_STUB_ATTR = "_source.stub";
+
+/**
+ * Node attribute: the entity a stub node is (a REST mapping's entity name).
+ * @category Network sources
+ */
+export const SOURCE_ENTITY_ATTR = "_source.entity";
+
+/**
+ * Node attribute: the node's degree at the source, when the source returns it at no extra cost
+ * (the Cypher source's degree hint).
+ * @category Network sources
+ */
+export const SOURCE_DEGREE_ATTR = "_source.degree";
+
+/**
+ * The prefix of `ImportGraphResult.format` for a load from a source: `"source:<id>"`.
+ * @category Network sources
+ */
+export const SOURCE_FORMAT_PREFIX = "source:";

package/src/sources/ndex/client.ts

@@ -0,0 +1,102 @@
+/**
+ * What every NDEx call shares: the checks on the endpoint and the network UUID, and a fetch that
+ * classifies NDEx's failures.
+ */
+
+import { failOn } from "../fail.js";
+import { type SourceContext } from "../types.js";
+
+/** An NDEx network UUID. */
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/** The query parameter NDEx reads a link-share access key from. */
+export const ACCESS_KEY_QUERY = "accesskey";
+
+/** The statuses whose JSON body (NDEx's errorCode) is read; every other failure is classified by status alone. */
+const READ_STATUSES = [401, 403, 404] as const;
+
+/**
+ * Refuse an access key in the endpoint: it belongs in a credential, which is never logged.
+ * @param ctx - the context
+ */
+export function checkEndpoint(ctx: SourceContext): void {
+    if (ctx.endpoint?.searchParams.has(ACCESS_KEY_QUERY) === true) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "an access key goes in a credential, never in the endpoint", {
+            name: "endpoint",
+            reason: "credentials-in-url",
+        });
+    }
+}
+
+/**
+ * Check a call's endpoint and network UUID.
+ * @param uuid - the network UUID
+ * @param ctx - the context
+ * @returns the UUID, lowercase and URL-safe
+ */
+export function checkedUuid(uuid: unknown, ctx: SourceContext): string {
+    checkEndpoint(ctx);
+    const value = String(uuid);
+    if (/accesskey/i.test(value)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "an access key goes in a credential, never in the uuid", {
+            name: "uuid",
+            reason: "credentials-in-url",
+        });
+    }
+    if (!UUID.test(value)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "the uuid is not a network UUID", { name: "uuid", reason: "format" });
+    }
+    return value.toLowerCase();
+}
+
+/**
+ * Fetch from NDEx, turning a 401, 403 or 404 into its coded error with NDEx's own errorCode.
+ * @param ctx - the context
+ * @param path - the path, relative to the endpoint
+ * @param uuid - the network the request is about, for E_SOURCE_NOT_FOUND
+ * @param init - the request init
+ * @returns the response, status 2xx
+ */
+export async function ndexFetch(
+    ctx: SourceContext,
+    path: string,
+    uuid: string | null,
+    init: RequestInit & { readonly idempotent?: boolean } = {},
+): Promise<Response> {
+    const response = await ctx.fetch(path, { ...init, allowStatus: READ_STATUSES });
+    if (response.ok) {
+        return response;
+    }
+    const url = new URL(path, ctx.endpoint ?? undefined);
+    const serverCode = await errorCode(response);
+    const { status } = response;
+    if (status === 401) {
+        failOn(ctx.report, "E_SOURCE_AUTH", `NDEx answered ${String(status)}`, {
+            origin: url.origin,
+            status,
+            scheme: response.headers.get("www-authenticate")?.split(/\s/)[0] ?? null,
+            serverCode,
+        });
+    }
+    if (status === 403) {
+        failOn(ctx.report, "E_SOURCE_FORBIDDEN", "NDEx answered 403", { status, serverCode });
+    }
+    return failOn(ctx.report, "E_SOURCE_NOT_FOUND", "NDEx has no such network", {
+        what: uuid === null ? "url" : "network",
+        value: uuid ?? `${url.origin}${url.pathname}`,
+    });
+}
+
+/**
+ * NDEx's own error code from a JSON error body, never its message or stack trace.
+ * @param response - the failed response
+ * @returns the code, or null
+ */
+async function errorCode(response: Response): Promise<string | null> {
+    try {
+        const body = (await response.json()) as { errorCode?: unknown };
+        return typeof body.errorCode === "string" ? body.errorCode.slice(0, 100) : null;
+    } catch {
+        return null;
+    }
+}

package/src/sources/ndex/descriptor.ts

@@ -0,0 +1,64 @@
+/**
+ * The NDEx source's descriptor: catalog data, available without loading the source's code.
+ */
+
+import { type SourceDescriptor, type SourceSample } from "../types.js";
+
+/**
+ * One public NDEx network offered as a sample.
+ * @param id - the sample id
+ * @param plainName - its name
+ * @param uuid - the network UUID
+ * @param nodes - its node count
+ * @param edges - its edge count
+ * @returns the sample
+ */
+function sample(id: string, plainName: string, uuid: string, nodes: number, edges: number): SourceSample {
+    return { id, plainName, params: { uuid }, counts: { nodes, edges } };
+}
+
+/**
+ * The descriptor of the `ndex` source. Experimental.
+ * @category Network sources
+ */
+export const NDEX_DESCRIPTOR: SourceDescriptor = Object.freeze({
+    plainName: "NDEx",
+    kind: "repository",
+    version: 1,
+    defaultEndpoint: "https://www.ndexbio.org/",
+    auth: ["basic", "bearer", "api-key"],
+    authRequired: "optional",
+    rateLimit: { requests: 50, perMs: 1000, concurrent: 4 },
+    params: [
+        { name: "uuid", plainName: "Network UUID", type: "string", required: true },
+        {
+            name: "format",
+            plainName: "Format",
+            type: "enum",
+            default: "cx2",
+            advanced: true,
+            values: [
+                { value: "cx2", label: "CX2" },
+                { value: "cx", label: "CX" },
+            ],
+        },
+        { name: "idAttribute", plainName: "Id attribute", type: "string", advanced: true },
+    ],
+    features: ["probe", "list", "expand", "validate"],
+    samples: [
+        sample("ndex-wp5483", "WikiPathways WP5483", "86719f5b-1439-11f0-9806-005056ae3c32", 57, 34),
+        sample("ndex-nci-pid", "NCI PID cancer signaling", "640e2cef-795d-11e8-a4bf-0ac135e8bacf", 2794, 25296),
+        sample("ndex-huri", "HuRI human interactome", "73bc2c06-5fb2-11e9-9f06-0ac135e8bacf", 8275, 52569),
+        sample("ndex-signor", "Signor", "523fff27-afe8-11e9-8bb4-0ac135e8bacf", 10339, 33381),
+        sample("ndex-dgidb", "DGIdb drugs and target genes", "7e5e64ff-f6cf-11ea-99da-0ac135e8bacf", 12100, 32814),
+        sample(
+            "ndex-wikipathways-landscape",
+            "WikiPathways landscape",
+            "2b3ed6af-5031-11eb-9e72-0ac135e8bacf",
+            32077,
+            74109,
+        ),
+        // 3.3 million edges and a 690 MB download: offered for neighborhoods, and as a maxBytes test
+        { ...sample("ndex-pcnet", "PCNet 2.2", "8b4b54fa-e87d-11ee-9621-005056ae23aa", 18558, 3323928), whole: false },
+    ],
+});

package/src/sources/ndex/expand.ts

@@ -0,0 +1,100 @@
+/**
+ * The NDEx source's expand: NDEx's neighborhood query (`POST /v2/search/network/{uuid}/query`),
+ * which answers CX1, read by the CX importer. Seeds are node ids, sent as `nodeIds`; NDEx's CX1
+ * answer keeps the ids of the CX2 download (checked on 2026-10-17), so an expansion merges into a
+ * loaded network by id. With `idAttribute`, seeds are values of that node attribute instead,
+ * matched by NDEx's node search.
+ */
+
+import { type GraphSink } from "@graphty/graph-format";
+
+import { cxImporter } from "../../formats/cx/importer.js";
+import { type ImportReport } from "../../types.js";
+import { failOn } from "../fail.js";
+import { type ExpandRequest, type SourceContext } from "../types.js";
+import { checkedUuid, ndexFetch } from "./client.js";
+import { type NdexParams } from "./params.js";
+
+/** The deepest neighborhood asked for, as for every built-in source. */
+const MAX_DEPTH = 10;
+
+/**
+ * Quote a value for NDEx's search syntax.
+ * @param value - the value
+ * @returns the quoted term
+ */
+function term(value: string): string {
+    const escaped = value.replaceAll(/["\\]/g, String.raw`\$&`);
+    return `"${escaped}"`;
+}
+
+/**
+ * The query body: the seeds as node ids, or as values of `idAttribute`.
+ * @param request - the expand request
+ * @param ctx - the context
+ * @returns the body NDEx's query takes
+ */
+function queryBody(request: ExpandRequest<NdexParams>, ctx: SourceContext): Record<string, unknown> {
+    const { ids } = request;
+    const { idAttribute } = request.params;
+    const depth = request.depth ?? 1;
+    if (!Number.isInteger(depth) || depth < 1 || depth > MAX_DEPTH) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "depth is an integer from 1 to 10", { name: "depth", reason: "range" });
+    }
+    if (ids.length === 0) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "an expansion needs at least one id", { name: "ids", reason: "required" });
+    }
+    const limit = request.limit === undefined ? {} : { edgeLimit: request.limit, errorWhenLimitIsOver: false };
+    if (idAttribute !== undefined) {
+        if (!/^[A-Za-z_][\w.]*$/.test(idAttribute)) {
+            failOn(ctx.report, "E_SOURCE_PARAM", "idAttribute is not an attribute name", {
+                name: "idAttribute",
+                reason: "identifier",
+            });
+        }
+        const searchString = ids.map((id) => `${idAttribute}:${term(String(id))}`).join(" ");
+        return { searchString, searchDepth: depth, ...limit };
+    }
+    const nodeIds = ids.map(Number);
+    if (nodeIds.some((n) => !Number.isSafeInteger(n))) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "NDEx node ids are integers; set idAttribute to seed by name", {
+            name: "ids",
+            reason: "type",
+        });
+    }
+    return { searchString: "", searchDepth: depth, nodeIds, ...limit };
+}
+
+/**
+ * Fetch the neighborhood of some nodes of an NDEx network into a sink.
+ * @param request - the seeds, depth and limit; `limit` caps the edges NDEx returns
+ * @param sink - where to write
+ * @param ctx - the context
+ * @returns the report
+ */
+export async function expandNdex(
+    request: ExpandRequest<NdexParams>,
+    sink: GraphSink,
+    ctx: SourceContext,
+): Promise<ImportReport> {
+    const uuid = checkedUuid(request.params.uuid, ctx);
+    if (request.direction !== undefined && request.direction !== "both") {
+        failOn(ctx.report, "E_SOURCE_UNSUPPORTED", "NDEx expands in both directions only", {
+            feature: "direction",
+            needs: "both",
+        });
+    }
+    if ((request.relationshipTypes ?? []).length > 0) {
+        failOn(ctx.report, "E_SOURCE_UNSUPPORTED", "NDEx cannot filter an expansion by relationship type", {
+            feature: "relationshipTypes",
+        });
+    }
+    const response = await ndexFetch(ctx, `v2/search/network/${uuid}/query`, uuid, {
+        method: "POST",
+        idempotent: true,
+        headers: { "Content-Type": "application/json", Accept: "application/json" },
+        body: JSON.stringify(queryBody(request, ctx)),
+    });
+    ctx.report.include(await cxImporter.import(response.body ?? "", sink, { signal: ctx.signal }));
+    return ctx.finish();
+}