@graphty/graph-io 0.3.48 → 0.3.49

package/src/sources/params.ts

@@ -6,6 +6,7 @@
 import { type CypherParams } from "./cypher/params.js";
 import { type NdexParams } from "./ndex/params.js";
 import { type OpenAlexParams, type RestParams } from "./rest/params.js";
+import { type SparqlParams } from "./sparql/params.js";
 
 /**
  * Parameter types by source id; extend it by declaration merging:
@@ -26,4 +27,6 @@ export interface SourceParamsMap {
     rest: RestParams;
     /** OpenAlex. */
     openalex: OpenAlexParams;
+    /** Any SPARQL 1.1 endpoint. */
+    sparql: SparqlParams;
 }

package/src/sources/rest/descriptor.ts

@@ -17,6 +17,8 @@ export const REST_DESCRIPTOR: SourceDescriptor = Object.freeze({
     version: 1,
     auth: ["bearer", "api-key"],
     authRequired: "optional",
+    // ponytail: a mapping without `expand` or paging cannot do either; per-mapping features when a form needs them
+    features: ["expand", "probe", "validate", "page"],
     params: [{ name: "mapping", plainName: "Mapping", type: "text", required: true }],
     samples: [
         {
@@ -47,7 +49,9 @@ export const OPENALEX_DESCRIPTOR: SourceDescriptor = Object.freeze({
     defaultEndpoint: "https://api.openalex.org",
     auth: ["api-key"],
     authRequired: "optional",
+    apiKey: { query: "api_key" },
     rateLimit: { requests: 10, perMs: 1000 },
+    features: ["expand", "probe", "validate", "page"],
     params: [
         { name: "work", plainName: "Work", type: "string" },
         { name: "author", plainName: "Author", type: "string" },
@@ -69,6 +73,7 @@ export const OPENALEX_DESCRIPTOR: SourceDescriptor = Object.freeze({
             plainName: "Watts-Strogatz paper's citations",
             params: { work: "W2112090702" },
             whole: false,
+            firstPage: 200,
         },
     ],
 });

package/src/sources/rest/source.ts

@@ -697,6 +697,21 @@ export type RestSourceInfo = {
     readonly id: string;
 } & Partial<SourceDescriptor>;
 
+/**
+ * Where a mapping's API key goes, as a descriptor says it.
+ * @param auth - the mapping's auth
+ * @returns `{ apiKey }`, or nothing when the mapping takes no key or does not say where
+ */
+function apiKeyOf(auth: RestMapping["auth"]): Pick<SourceDescriptor, "apiKey"> {
+    if (auth?.kind !== "api-key") {
+        return {};
+    }
+    if (auth.header !== undefined) {
+        return { apiKey: { header: auth.header } };
+    }
+    return auth.query === undefined ? {} : { apiKey: { query: auth.query } };
+}
+
 /**
  * Turn a REST mapping into a source: a `records` source that loads the mapping's `load` request
  * page by page, with `expand` (breadth-first over the `expand` requests, `{ids}` batched; depth 0
@@ -733,6 +748,7 @@ export function restSource(mapping: RestMapping, info: RestSourceInfo): GraphSou
         defaultEndpoint: mapping.endpoint,
         auth,
         authRequired: auth.length === 0 ? "none" : "optional",
+        ...apiKeyOf(mapping.auth),
         ...(mapping.rateLimit === undefined ? {} : { rateLimit: mapping.rateLimit }),
         ...fields,
     });

package/src/sources/sigv4.ts

@@ -75,12 +75,14 @@ async function sha256(data: Uint8Array | ArrayBuffer): Promise<string> {
 }
 
 /**
- * SHA-256 of a text's UTF-8 bytes, as lowercase hex (the Cypher source's content hash).
- * @param text - the text
+ * SHA-256 as 64 lowercase hexadecimal digits: of bytes exactly as given, or of a text's UTF-8 bytes
+ * (the Cypher source's content hash, a project file's record of a file it loaded). Web Crypto where
+ * there is one, else plain JavaScript (a page that is not a secure context). Experimental.
+ * @param data - the bytes or the text
  * @returns the digest
  */
-export function sha256Hex(text: string): Promise<string> {
-    return sha256(utf8(text));
+export function sha256Hex(data: string | Uint8Array): Promise<string> {
+    return sha256(typeof data === "string" ? utf8(data) : data);
 }
 
 /**

package/src/sources/sparql/descriptor.ts

@@ -0,0 +1,67 @@
+/**
+ * The SPARQL source's descriptor: catalog data, available without loading the source's code.
+ */
+
+import { type SourceDescriptor } from "../types.js";
+
+/** A small Wikidata query: physics Nobel laureates and their doctoral advisors. */
+const WIKIDATA_ADVISORS = `PREFIX wd: <http://www.wikidata.org/entity/>
+PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+CONSTRUCT { ?person wdt:P184 ?advisor . ?person rdfs:label ?name . ?advisor rdfs:label ?advisorName . }
+WHERE {
+  ?person wdt:P166 wd:Q38104 ; wdt:P184 ?advisor ; rdfs:label ?name .
+  ?advisor rdfs:label ?advisorName .
+  FILTER(lang(?name) = "en" && lang(?advisorName) = "en")
+}`;
+
+/**
+ * The descriptor of the `sparql` source. Experimental.
+ * @category Network sources
+ */
+export const SPARQL_DESCRIPTOR: SourceDescriptor = Object.freeze({
+    plainName: "SPARQL endpoint",
+    kind: "sparql",
+    version: 1,
+    auth: ["basic", "bearer"],
+    authRequired: "optional",
+    features: ["expand", "probe", "validate", "page"],
+    params: [
+        { name: "query", plainName: "Query", type: "query", language: "sparql" },
+        {
+            name: "mode",
+            plainName: "Mode",
+            type: "enum",
+            values: [
+                { value: "query", label: "Query" },
+                { value: "all", label: "Everything" },
+                { value: "neighborhood", label: "Seed nodes" },
+            ],
+        },
+        { name: "ids", plainName: "Seed IRIs", type: "node-id" },
+        { name: "language", plainName: "Language", type: "string", advanced: true },
+        { name: "predicates", plainName: "Follow predicates", type: "string", advanced: true },
+        {
+            name: "pageSize",
+            plainName: "Page size",
+            type: "integer",
+            default: 1000,
+            min: 1,
+            max: 10000,
+            advanced: true,
+        },
+    ],
+    samples: [
+        {
+            id: "wikidata-physics-advisors",
+            plainName: "Wikidata: physics Nobel laureates and their advisors",
+            endpoint: "https://query.wikidata.org/sparql",
+            params: {
+                query: WIKIDATA_ADVISORS,
+                pageSize: 100,
+                language: "en",
+                predicates: ["http://www.wikidata.org/prop/direct/"], // NOSONAR(S5332): an RDF namespace IRI, never fetched; Wikidata's IRIs are http
+            },
+        },
+    ],
+});

package/src/sources/sparql/index.ts

@@ -0,0 +1,356 @@
+/**
+ * `@graphty/graph-io/sources/sparql`: the SPARQL source. It sends CONSTRUCT (or DESCRIBE) queries
+ * over the SPARQL 1.1 protocol (a form POST, N-Triples back) and reads the triples as a graph: an
+ * IRI or blank-node object is an edge from the subject (`type` is the predicate's local name,
+ * `predicate` the full IRI), a literal is an attribute of the subject named by the predicate's
+ * local name. Node ids are the IRIs.
+ *
+ * - Paging: the source adds `ORDER BY` (the template's variables) and `LIMIT` / `OFFSET` to the
+ *   caller's query, unless it sets its own; a load stops between pages at a cap and hands back
+ *   `{ offset }` as its cursor. A page is `pageSize` solutions, so its triple count varies; the
+ *   first empty page ends the load.
+ * - Expand: the triples of the seeds, `VALUES ?s { ... }` out and `VALUES ?o { ... }` in.
+ * - Validate: SHA-256 of the first page's triples.
+ *
+ * Experimental.
+ * @module @graphty/graph-io/sources/sparql
+ */
+
+import { type GraphSink } from "@graphty/graph-format";
+
+import { throwIfAborted } from "../../common/input.js";
+import { type ImportReport } from "../../types.js";
+import { defineSource } from "../define.js";
+import { failOn } from "../fail.js";
+import { SPARQL_SOURCE_ID } from "../ids.js";
+import { sha256Hex } from "../sigv4.js";
+import {
+    type ExpandRequest,
+    type GraphSource,
+    type SourceContext,
+    type SourceCursor,
+    type SourceFreshness,
+    type SourceProbe,
+} from "../types.js";
+import { SPARQL_DESCRIPTOR } from "./descriptor.js";
+import { canonicalTriple, type Graphed, graphTriples, parseNTriples, type Triple } from "./ntriples.js";
+import { type SparqlParams } from "./params.js";
+import { allQuery, aroundQuery, iriTerm, pagedQuery, readQuery, validLanguage } from "./query.js";
+
+export { SPARQL_DESCRIPTOR } from "./descriptor.js";
+export { type SparqlParams } from "./params.js";
+
+/** Solutions per page when the params name none. */
+const DEFAULT_PAGE_SIZE = 1000;
+
+/** Triples per hop of an expansion when the request names no limit. */
+const DEFAULT_EXPAND_LIMIT = 500;
+
+/** The deepest expansion, as for every built-in source. */
+const MAX_DEPTH = 10;
+
+/** What CONSTRUCT and DESCRIBE are asked to answer. */
+const ACCEPT_TRIPLES = "application/n-triples, text/plain;q=0.9";
+
+/** One page's query, by offset; `paged` false when the query is sent once, as written. */
+interface Plan {
+    /** The query of the page at an offset. */
+    readonly at: (offset: number) => string;
+    /** Whether the source pages it. */
+    readonly paged: boolean;
+}
+
+/**
+ * Send a query and read the answer as N-Triples.
+ * @param ctx - the context
+ * @param query - the query
+ * @returns the triples
+ */
+async function send(ctx: SourceContext, query: string): Promise<Triple[]> {
+    throwIfAborted(ctx.signal);
+    const response = await ctx.fetch(ctx.endpoint ?? "", {
+        method: "POST",
+        idempotent: true,
+        headers: { Accept: ACCEPT_TRIPLES, "Content-Type": "application/x-www-form-urlencoded" },
+        body: new URLSearchParams({ query }).toString(),
+        allowStatus: [400],
+    });
+    if (response.status === 400) {
+        const text = (await response.text()).trim().slice(0, 500);
+        failOn(ctx.report, "E_SOURCE_QUERY", "the endpoint refused the query", {
+            serverCode: null,
+            line: null,
+            column: null,
+            serverMessage: text === "" ? null : text,
+        });
+    }
+    const contentType = response.headers.get("content-type");
+    const notTriples = (): never =>
+        failOn(ctx.report, "E_SOURCE_RESPONSE", "the endpoint did not answer N-Triples", {
+            expected: "n-triples",
+            contentType,
+        });
+    if (contentType !== null && /html|json|xml/i.test(contentType)) {
+        notTriples();
+    }
+    const { triples, badLine } = parseNTriples(await response.text());
+    if (badLine !== null) {
+        notTriples();
+    }
+    return triples;
+}
+
+/**
+ * A whole-number param, checked.
+ * @param value - the value
+ * @param name - the param
+ * @param max - the largest allowed
+ * @param ctx - the context
+ * @returns the number
+ */
+function integer(value: number, name: string, max: number, ctx: SourceContext): number {
+    if (!Number.isInteger(value) || value < 1 || value > max) {
+        failOn(ctx.report, "E_SOURCE_PARAM", `${name} is an integer from 1 to ${String(max)}`, {
+            name,
+            reason: "range",
+        });
+    }
+    return value;
+}
+
+/**
+ * The language filter, lowercase and checked, or "" for none.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the tag
+ */
+function languageOf(params: SparqlParams, ctx: SourceContext): string {
+    const language = params.language ?? "";
+    if (language !== "" && !validLanguage(language)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "language is a language tag", { name: "language", reason: "type" });
+    }
+    return language.toLowerCase();
+}
+
+/**
+ * The predicate prefixes generated queries follow, checked.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the prefixes; empty means every predicate
+ */
+function predicatesOf(params: SparqlParams, ctx: SourceContext): readonly string[] {
+    const list = params.predicates ?? [];
+    const prefixes = typeof list === "string" ? [list] : list;
+    for (const p of prefixes) {
+        if (typeof p !== "string" || iriTerm(p) === null) {
+            failOn(ctx.report, "E_SOURCE_PARAM", "predicates are IRI prefixes", { name: "predicates", reason: "iri" });
+        }
+    }
+    return prefixes;
+}
+
+/**
+ * Seed ids as IRI terms; a blank node or anything else that is not an IRI is refused.
+ * @param ids - the ids
+ * @param name - the param, for the error
+ * @param ctx - the context
+ * @returns the terms
+ */
+function irisOf(ids: readonly unknown[] | undefined, name: string, ctx: SourceContext): string[] {
+    if (ids === undefined || ids.length === 0) {
+        failOn(ctx.report, "E_SOURCE_PARAM", `${name} needs at least one IRI`, { name, reason: "required" });
+    }
+    return ids.map(
+        (id) => iriTerm(id) ?? failOn(ctx.report, "E_SOURCE_PARAM", `${name}: not an IRI`, { name, reason: "iri" }),
+    );
+}
+
+/**
+ * What a load sends, from its mode.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the plan
+ */
+function planOf(params: SparqlParams, ctx: SourceContext): Plan {
+    const size = integer(params.pageSize ?? DEFAULT_PAGE_SIZE, "pageSize", 10_000, ctx);
+    const filter = { language: languageOf(params, ctx), predicates: predicatesOf(params, ctx) };
+    const query = params.query?.trim() ?? "";
+    const mode = params.mode ?? (query === "" ? "all" : "query");
+    if (mode === "neighborhood") {
+        const iris = irisOf(params.ids, "ids", ctx);
+        return { at: (offset) => aroundQuery(iris, "both", filter, size, offset), paged: true };
+    }
+    if (mode === "all") {
+        return { at: (offset) => allQuery(filter, size, offset), paged: true };
+    }
+    if (query === "") {
+        failOn(ctx.report, "E_SOURCE_PARAM", "mode query needs a query", { name: "query", reason: "required" });
+    }
+    const shape = readQuery(query);
+    if (shape.form !== "construct" && shape.form !== "describe") {
+        failOn(ctx.report, "E_SOURCE_PARAM", "the query must be CONSTRUCT or DESCRIBE", {
+            name: "query",
+            reason: "form",
+        });
+    }
+    if (shape.ownLimit) {
+        return { at: () => query, paged: false };
+    }
+    return { at: (offset) => pagedQuery(query, shape, size, offset), paged: true };
+}
+
+/**
+ * Write nodes and edges into the sink, counting them on the report.
+ * @param graphed - the nodes and edges
+ * @param sink - where to write
+ * @param ctx - the context
+ */
+function write(graphed: Graphed, sink: GraphSink, ctx: SourceContext): void {
+    const { report } = ctx;
+    if (sink.edgeCount === 0) {
+        sink.setDirected(true);
+    }
+    for (const [id, attrs] of graphed.nodes) {
+        try {
+            sink.addNodeRecord(id, attrs);
+            report.counts.nodes++;
+        } catch (err) {
+            report.recordError(err, { element: id });
+            report.counts.skippedNodes++;
+        }
+    }
+    for (const { source, target, ...attrs } of graphed.edges) {
+        try {
+            sink.addEdgeRecord(source, target, attrs, null);
+            report.counts.edges++;
+        } catch (err) {
+            report.recordError(err, { element: `${source} -> ${target}` });
+            report.counts.skippedEdges++;
+        }
+    }
+}
+
+/**
+ * The content hash of some triples, in any order.
+ * @param triples - the triples
+ * @returns the SHA-256, hex
+ */
+function hashOf(triples: readonly Triple[]): Promise<string> {
+    return sha256Hex(triples.map(canonicalTriple).sort().join("\n")); // NOSONAR(S2871): code-unit order is the point: a locale-free, stable order for the hash
+}
+
+/**
+ * Load pages into the sink from the cursor's offset until an empty page or a cap.
+ * @param params - the params
+ * @param sink - where to write
+ * @param ctx - the context
+ * @returns the report
+ */
+async function load(params: SparqlParams, sink: GraphSink, ctx: SourceContext): Promise<ImportReport> {
+    const plan = planOf(params, ctx);
+    const size = params.pageSize ?? DEFAULT_PAGE_SIZE;
+    const start = Number(ctx.cursor?.offset ?? 0);
+    const language = languageOf(params, ctx);
+    let offset = start;
+    let next: SourceCursor | null = null;
+    let contentHash: string | undefined;
+    for (;;) {
+        const triples = await send(ctx, plan.at(offset));
+        if (offset === 0) {
+            contentHash = await hashOf(triples);
+        }
+        write(graphTriples(triples, language), sink, ctx);
+        const { counts } = ctx.report;
+        ctx.progress(counts.nodes + counts.edges, undefined, "items");
+        if (!plan.paged || triples.length === 0) {
+            next = null;
+            break;
+        }
+        offset += size;
+        next = { offset };
+        if (ctx.limits.reached()) {
+            break;
+        }
+    }
+    return ctx.finish({ next, ...(contentHash === undefined ? {} : { validators: { contentHash } }) });
+}
+
+/**
+ * Fetch the triples around some IRIs into the sink, hop by hop.
+ * @param request - the seeds, direction, depth, limit and predicate filter
+ * @param sink - where to write
+ * @param ctx - the context
+ * @returns the report
+ */
+async function expand(
+    request: ExpandRequest<SparqlParams>,
+    sink: GraphSink,
+    ctx: SourceContext,
+): Promise<ImportReport> {
+    const direction = request.direction ?? "both";
+    if (!["out", "in", "both"].includes(direction)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "direction must be out, in or both", {
+            name: "direction",
+            reason: "enum",
+        });
+    }
+    const depth = integer(request.depth ?? 1, "depth", MAX_DEPTH, ctx);
+    const limit = integer(request.limit ?? DEFAULT_EXPAND_LIMIT, "limit", Number.MAX_SAFE_INTEGER, ctx);
+    const language = languageOf(request.params, ctx);
+    const filter = { language, predicates: predicatesOf(request.params, ctx) };
+    const types = request.relationshipTypes === undefined ? null : new Set(request.relationshipTypes);
+    // a capped one-hop expansion resumes at its offset; deeper ones are read whole per hop
+    const offset = depth === 1 ? Number(ctx.cursor?.offset ?? 0) : 0;
+    const seen = new Set<string>(request.ids.map(String));
+    let frontier = irisOf(request.ids, "ids", ctx);
+    let next: SourceCursor | null = null;
+    const graphed: Graphed = { nodes: new Map(), edges: [] };
+    for (let hop = 0; hop < depth && frontier.length > 0; hop++) {
+        const triples = await send(ctx, aroundQuery(frontier, direction, filter, limit, offset));
+        const before = graphed.edges.length;
+        graphTriples(triples, language, graphed);
+        if (types !== null) {
+            const kept = graphed.edges.slice(before).filter((e) => types.has(e.type) || types.has(e.predicate));
+            graphed.edges.splice(before, Infinity, ...kept);
+        }
+        if (depth === 1 && triples.length > 0) {
+            next = { offset: offset + limit };
+        }
+        const reached = graphed.edges.slice(before).flatMap((e) => [e.source, e.target]);
+        frontier = [...new Set(reached)]
+            .filter((id) => !seen.has(id))
+            .flatMap((id) => {
+                seen.add(id);
+                const term = iriTerm(id);
+                return term === null ? [] : [term];
+            });
+    }
+    write(graphed, sink, ctx);
+    return ctx.finish({ next });
+}
+
+/**
+ * The SPARQL source: CONSTRUCT or DESCRIBE queries against any SPARQL 1.1 endpoint, paged by
+ * `LIMIT` and `OFFSET`, expanded by `VALUES`, validated by a hash of the first page. Experimental.
+ * @category Network sources
+ */
+export const sparqlSource: GraphSource<SparqlParams> = defineSource<SparqlParams>({
+    id: SPARQL_SOURCE_ID,
+    descriptor: SPARQL_DESCRIPTOR,
+    import: load,
+    expand,
+    probe: async (params, ctx): Promise<SourceProbe> => {
+        // the endpoint answers a one-solution query: it is up, it reads SPARQL, the credential works
+        await send(ctx, "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1");
+        languageOf(params, ctx);
+        return { ok: true };
+    },
+    validate: async (params, seen, ctx): Promise<SourceFreshness> => {
+        const contentHash = await hashOf(await send(ctx, planOf(params, ctx).at(0)));
+        let status: SourceFreshness["status"] = "unverifiable";
+        if (seen.contentHash !== undefined) {
+            status = seen.contentHash === contentHash ? "current" : "changed";
+        }
+        return { status, via: "hash", validators: { contentHash } };
+    },
+});