@graphty/graph-io 0.3.47 → 0.3.49

package/src/sources/sparql/index.ts

@@ -0,0 +1,356 @@
+/**
+ * `@graphty/graph-io/sources/sparql`: the SPARQL source. It sends CONSTRUCT (or DESCRIBE) queries
+ * over the SPARQL 1.1 protocol (a form POST, N-Triples back) and reads the triples as a graph: an
+ * IRI or blank-node object is an edge from the subject (`type` is the predicate's local name,
+ * `predicate` the full IRI), a literal is an attribute of the subject named by the predicate's
+ * local name. Node ids are the IRIs.
+ *
+ * - Paging: the source adds `ORDER BY` (the template's variables) and `LIMIT` / `OFFSET` to the
+ *   caller's query, unless it sets its own; a load stops between pages at a cap and hands back
+ *   `{ offset }` as its cursor. A page is `pageSize` solutions, so its triple count varies; the
+ *   first empty page ends the load.
+ * - Expand: the triples of the seeds, `VALUES ?s { ... }` out and `VALUES ?o { ... }` in.
+ * - Validate: SHA-256 of the first page's triples.
+ *
+ * Experimental.
+ * @module @graphty/graph-io/sources/sparql
+ */
+
+import { type GraphSink } from "@graphty/graph-format";
+
+import { throwIfAborted } from "../../common/input.js";
+import { type ImportReport } from "../../types.js";
+import { defineSource } from "../define.js";
+import { failOn } from "../fail.js";
+import { SPARQL_SOURCE_ID } from "../ids.js";
+import { sha256Hex } from "../sigv4.js";
+import {
+    type ExpandRequest,
+    type GraphSource,
+    type SourceContext,
+    type SourceCursor,
+    type SourceFreshness,
+    type SourceProbe,
+} from "../types.js";
+import { SPARQL_DESCRIPTOR } from "./descriptor.js";
+import { canonicalTriple, type Graphed, graphTriples, parseNTriples, type Triple } from "./ntriples.js";
+import { type SparqlParams } from "./params.js";
+import { allQuery, aroundQuery, iriTerm, pagedQuery, readQuery, validLanguage } from "./query.js";
+
+export { SPARQL_DESCRIPTOR } from "./descriptor.js";
+export { type SparqlParams } from "./params.js";
+
+/** Solutions per page when the params name none. */
+const DEFAULT_PAGE_SIZE = 1000;
+
+/** Triples per hop of an expansion when the request names no limit. */
+const DEFAULT_EXPAND_LIMIT = 500;
+
+/** The deepest expansion, as for every built-in source. */
+const MAX_DEPTH = 10;
+
+/** What CONSTRUCT and DESCRIBE are asked to answer. */
+const ACCEPT_TRIPLES = "application/n-triples, text/plain;q=0.9";
+
+/** One page's query, by offset; `paged` false when the query is sent once, as written. */
+interface Plan {
+    /** The query of the page at an offset. */
+    readonly at: (offset: number) => string;
+    /** Whether the source pages it. */
+    readonly paged: boolean;
+}
+
+/**
+ * Send a query and read the answer as N-Triples.
+ * @param ctx - the context
+ * @param query - the query
+ * @returns the triples
+ */
+async function send(ctx: SourceContext, query: string): Promise<Triple[]> {
+    throwIfAborted(ctx.signal);
+    const response = await ctx.fetch(ctx.endpoint ?? "", {
+        method: "POST",
+        idempotent: true,
+        headers: { Accept: ACCEPT_TRIPLES, "Content-Type": "application/x-www-form-urlencoded" },
+        body: new URLSearchParams({ query }).toString(),
+        allowStatus: [400],
+    });
+    if (response.status === 400) {
+        const text = (await response.text()).trim().slice(0, 500);
+        failOn(ctx.report, "E_SOURCE_QUERY", "the endpoint refused the query", {
+            serverCode: null,
+            line: null,
+            column: null,
+            serverMessage: text === "" ? null : text,
+        });
+    }
+    const contentType = response.headers.get("content-type");
+    const notTriples = (): never =>
+        failOn(ctx.report, "E_SOURCE_RESPONSE", "the endpoint did not answer N-Triples", {
+            expected: "n-triples",
+            contentType,
+        });
+    if (contentType !== null && /html|json|xml/i.test(contentType)) {
+        notTriples();
+    }
+    const { triples, badLine } = parseNTriples(await response.text());
+    if (badLine !== null) {
+        notTriples();
+    }
+    return triples;
+}
+
+/**
+ * A whole-number param, checked.
+ * @param value - the value
+ * @param name - the param
+ * @param max - the largest allowed
+ * @param ctx - the context
+ * @returns the number
+ */
+function integer(value: number, name: string, max: number, ctx: SourceContext): number {
+    if (!Number.isInteger(value) || value < 1 || value > max) {
+        failOn(ctx.report, "E_SOURCE_PARAM", `${name} is an integer from 1 to ${String(max)}`, {
+            name,
+            reason: "range",
+        });
+    }
+    return value;
+}
+
+/**
+ * The language filter, lowercase and checked, or "" for none.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the tag
+ */
+function languageOf(params: SparqlParams, ctx: SourceContext): string {
+    const language = params.language ?? "";
+    if (language !== "" && !validLanguage(language)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "language is a language tag", { name: "language", reason: "type" });
+    }
+    return language.toLowerCase();
+}
+
+/**
+ * The predicate prefixes generated queries follow, checked.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the prefixes; empty means every predicate
+ */
+function predicatesOf(params: SparqlParams, ctx: SourceContext): readonly string[] {
+    const list = params.predicates ?? [];
+    const prefixes = typeof list === "string" ? [list] : list;
+    for (const p of prefixes) {
+        if (typeof p !== "string" || iriTerm(p) === null) {
+            failOn(ctx.report, "E_SOURCE_PARAM", "predicates are IRI prefixes", { name: "predicates", reason: "iri" });
+        }
+    }
+    return prefixes;
+}
+
+/**
+ * Seed ids as IRI terms; a blank node or anything else that is not an IRI is refused.
+ * @param ids - the ids
+ * @param name - the param, for the error
+ * @param ctx - the context
+ * @returns the terms
+ */
+function irisOf(ids: readonly unknown[] | undefined, name: string, ctx: SourceContext): string[] {
+    if (ids === undefined || ids.length === 0) {
+        failOn(ctx.report, "E_SOURCE_PARAM", `${name} needs at least one IRI`, { name, reason: "required" });
+    }
+    return ids.map(
+        (id) => iriTerm(id) ?? failOn(ctx.report, "E_SOURCE_PARAM", `${name}: not an IRI`, { name, reason: "iri" }),
+    );
+}
+
+/**
+ * What a load sends, from its mode.
+ * @param params - the params
+ * @param ctx - the context
+ * @returns the plan
+ */
+function planOf(params: SparqlParams, ctx: SourceContext): Plan {
+    const size = integer(params.pageSize ?? DEFAULT_PAGE_SIZE, "pageSize", 10_000, ctx);
+    const filter = { language: languageOf(params, ctx), predicates: predicatesOf(params, ctx) };
+    const query = params.query?.trim() ?? "";
+    const mode = params.mode ?? (query === "" ? "all" : "query");
+    if (mode === "neighborhood") {
+        const iris = irisOf(params.ids, "ids", ctx);
+        return { at: (offset) => aroundQuery(iris, "both", filter, size, offset), paged: true };
+    }
+    if (mode === "all") {
+        return { at: (offset) => allQuery(filter, size, offset), paged: true };
+    }
+    if (query === "") {
+        failOn(ctx.report, "E_SOURCE_PARAM", "mode query needs a query", { name: "query", reason: "required" });
+    }
+    const shape = readQuery(query);
+    if (shape.form !== "construct" && shape.form !== "describe") {
+        failOn(ctx.report, "E_SOURCE_PARAM", "the query must be CONSTRUCT or DESCRIBE", {
+            name: "query",
+            reason: "form",
+        });
+    }
+    if (shape.ownLimit) {
+        return { at: () => query, paged: false };
+    }
+    return { at: (offset) => pagedQuery(query, shape, size, offset), paged: true };
+}
+
+/**
+ * Write nodes and edges into the sink, counting them on the report.
+ * @param graphed - the nodes and edges
+ * @param sink - where to write
+ * @param ctx - the context
+ */
+function write(graphed: Graphed, sink: GraphSink, ctx: SourceContext): void {
+    const { report } = ctx;
+    if (sink.edgeCount === 0) {
+        sink.setDirected(true);
+    }
+    for (const [id, attrs] of graphed.nodes) {
+        try {
+            sink.addNodeRecord(id, attrs);
+            report.counts.nodes++;
+        } catch (err) {
+            report.recordError(err, { element: id });
+            report.counts.skippedNodes++;
+        }
+    }
+    for (const { source, target, ...attrs } of graphed.edges) {
+        try {
+            sink.addEdgeRecord(source, target, attrs, null);
+            report.counts.edges++;
+        } catch (err) {
+            report.recordError(err, { element: `${source} -> ${target}` });
+            report.counts.skippedEdges++;
+        }
+    }
+}
+
+/**
+ * The content hash of some triples, in any order.
+ * @param triples - the triples
+ * @returns the SHA-256, hex
+ */
+function hashOf(triples: readonly Triple[]): Promise<string> {
+    return sha256Hex(triples.map(canonicalTriple).sort().join("\n")); // NOSONAR(S2871): code-unit order is the point: a locale-free, stable order for the hash
+}
+
+/**
+ * Load pages into the sink from the cursor's offset until an empty page or a cap.
+ * @param params - the params
+ * @param sink - where to write
+ * @param ctx - the context
+ * @returns the report
+ */
+async function load(params: SparqlParams, sink: GraphSink, ctx: SourceContext): Promise<ImportReport> {
+    const plan = planOf(params, ctx);
+    const size = params.pageSize ?? DEFAULT_PAGE_SIZE;
+    const start = Number(ctx.cursor?.offset ?? 0);
+    const language = languageOf(params, ctx);
+    let offset = start;
+    let next: SourceCursor | null = null;
+    let contentHash: string | undefined;
+    for (;;) {
+        const triples = await send(ctx, plan.at(offset));
+        if (offset === 0) {
+            contentHash = await hashOf(triples);
+        }
+        write(graphTriples(triples, language), sink, ctx);
+        const { counts } = ctx.report;
+        ctx.progress(counts.nodes + counts.edges, undefined, "items");
+        if (!plan.paged || triples.length === 0) {
+            next = null;
+            break;
+        }
+        offset += size;
+        next = { offset };
+        if (ctx.limits.reached()) {
+            break;
+        }
+    }
+    return ctx.finish({ next, ...(contentHash === undefined ? {} : { validators: { contentHash } }) });
+}
+
+/**
+ * Fetch the triples around some IRIs into the sink, hop by hop.
+ * @param request - the seeds, direction, depth, limit and predicate filter
+ * @param sink - where to write
+ * @param ctx - the context
+ * @returns the report
+ */
+async function expand(
+    request: ExpandRequest<SparqlParams>,
+    sink: GraphSink,
+    ctx: SourceContext,
+): Promise<ImportReport> {
+    const direction = request.direction ?? "both";
+    if (!["out", "in", "both"].includes(direction)) {
+        failOn(ctx.report, "E_SOURCE_PARAM", "direction must be out, in or both", {
+            name: "direction",
+            reason: "enum",
+        });
+    }
+    const depth = integer(request.depth ?? 1, "depth", MAX_DEPTH, ctx);
+    const limit = integer(request.limit ?? DEFAULT_EXPAND_LIMIT, "limit", Number.MAX_SAFE_INTEGER, ctx);
+    const language = languageOf(request.params, ctx);
+    const filter = { language, predicates: predicatesOf(request.params, ctx) };
+    const types = request.relationshipTypes === undefined ? null : new Set(request.relationshipTypes);
+    // a capped one-hop expansion resumes at its offset; deeper ones are read whole per hop
+    const offset = depth === 1 ? Number(ctx.cursor?.offset ?? 0) : 0;
+    const seen = new Set<string>(request.ids.map(String));
+    let frontier = irisOf(request.ids, "ids", ctx);
+    let next: SourceCursor | null = null;
+    const graphed: Graphed = { nodes: new Map(), edges: [] };
+    for (let hop = 0; hop < depth && frontier.length > 0; hop++) {
+        const triples = await send(ctx, aroundQuery(frontier, direction, filter, limit, offset));
+        const before = graphed.edges.length;
+        graphTriples(triples, language, graphed);
+        if (types !== null) {
+            const kept = graphed.edges.slice(before).filter((e) => types.has(e.type) || types.has(e.predicate));
+            graphed.edges.splice(before, Infinity, ...kept);
+        }
+        if (depth === 1 && triples.length > 0) {
+            next = { offset: offset + limit };
+        }
+        const reached = graphed.edges.slice(before).flatMap((e) => [e.source, e.target]);
+        frontier = [...new Set(reached)]
+            .filter((id) => !seen.has(id))
+            .flatMap((id) => {
+                seen.add(id);
+                const term = iriTerm(id);
+                return term === null ? [] : [term];
+            });
+    }
+    write(graphed, sink, ctx);
+    return ctx.finish({ next });
+}
+
+/**
+ * The SPARQL source: CONSTRUCT or DESCRIBE queries against any SPARQL 1.1 endpoint, paged by
+ * `LIMIT` and `OFFSET`, expanded by `VALUES`, validated by a hash of the first page. Experimental.
+ * @category Network sources
+ */
+export const sparqlSource: GraphSource<SparqlParams> = defineSource<SparqlParams>({
+    id: SPARQL_SOURCE_ID,
+    descriptor: SPARQL_DESCRIPTOR,
+    import: load,
+    expand,
+    probe: async (params, ctx): Promise<SourceProbe> => {
+        // the endpoint answers a one-solution query: it is up, it reads SPARQL, the credential works
+        await send(ctx, "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1");
+        languageOf(params, ctx);
+        return { ok: true };
+    },
+    validate: async (params, seen, ctx): Promise<SourceFreshness> => {
+        const contentHash = await hashOf(await send(ctx, planOf(params, ctx).at(0)));
+        let status: SourceFreshness["status"] = "unverifiable";
+        if (seen.contentHash !== undefined) {
+            status = seen.contentHash === contentHash ? "current" : "changed";
+        }
+        return { status, via: "hash", validators: { contentHash } };
+    },
+});

package/src/sources/sparql/ntriples.ts

@@ -0,0 +1,219 @@
+/**
+ * Reading the N-Triples a CONSTRUCT or DESCRIBE answers, and turning triples into nodes and edges:
+ * an IRI or blank-node object is an edge from the subject, a literal is an attribute of the
+ * subject named by the predicate's local name.
+ */
+
+/** One RDF term. */
+export type Term =
+    | { readonly kind: "iri"; readonly value: string }
+    | { readonly kind: "bnode"; readonly value: string }
+    | { readonly kind: "literal"; readonly value: string; readonly lang: string; readonly datatype: string };
+
+/** One triple. */
+export interface Triple {
+    /** The subject: an IRI or a blank node. */
+    readonly s: Term;
+    /** The predicate IRI. */
+    readonly p: string;
+    /** The object. */
+    readonly o: Term;
+}
+
+/** One term at the sticky position: an IRI, a blank node, or a literal with its tag or type. */
+const TERM = /\s*(?:<([^<>]*)>|_:([\w.-]*[\w-])|"((?:[^"\\]|\\.)*)"(?:@([A-Za-z][\w-]*)|\^\^<([^<>]*)>)?)/y;
+
+/** The end of a statement: a dot, then an optional comment. */
+const END = /\s*\.\s*(?:#.*)?$/y;
+
+/** The XSD namespace. */
+const XSD = "http://www.w3.org/2001/XMLSchema#";
+
+/** The XSD numeric types, read as numbers. */
+const NUMERIC: ReadonlySet<string> = new Set(
+    [
+        "integer",
+        "decimal",
+        "double",
+        "float",
+        "int",
+        "long",
+        "short",
+        "byte",
+        "nonNegativeInteger",
+        "nonPositiveInteger",
+        "positiveInteger",
+        "negativeInteger",
+        "unsignedInt",
+        "unsignedLong",
+        "unsignedShort",
+        "unsignedByte",
+    ].map((t) => XSD + t),
+);
+
+/**
+ * Undo N-Triples escapes.
+ * @param text - escaped text
+ * @returns the text
+ */
+function unescape(text: string): string {
+    if (!text.includes("\\")) {
+        return text;
+    }
+    const simple: Readonly<Record<string, string>> = { t: "\t", b: "\b", n: "\n", r: "\r", f: "\f" };
+    return text.replaceAll(
+        /\\(?:u([\da-fA-F]{4})|U([\da-fA-F]{8})|(.))/g,
+        (_m, u4?: string, u8?: string, c?: string) => {
+            const hex = u4 ?? u8;
+            if (hex !== undefined) {
+                return String.fromCodePoint(Number.parseInt(hex, 16));
+            }
+            return simple[c ?? ""] ?? c ?? "";
+        },
+    );
+}
+
+/**
+ * Read one term at `at`.
+ * @param line - the line
+ * @param at - where to start
+ * @returns the term and where it ended, or null when there is none
+ */
+function readTerm(line: string, at: number): { term: Term; end: number } | null {
+    TERM.lastIndex = at;
+    const m = TERM.exec(line);
+    if (m === null) {
+        return null;
+    }
+    const [, iri, bnode, lexical, lang, datatype] = m;
+    let term: Term;
+    if (iri !== undefined) {
+        term = { kind: "iri", value: unescape(iri) };
+    } else if (bnode === undefined) {
+        term = {
+            kind: "literal",
+            value: unescape(lexical ?? ""),
+            lang: lang?.toLowerCase() ?? "",
+            datatype: datatype ?? "",
+        };
+    } else {
+        term = { kind: "bnode", value: `_:${bnode}` };
+    }
+    return { term, end: TERM.lastIndex };
+}
+
+/**
+ * Parse N-Triples.
+ * @param text - the document
+ * @returns the triples, or the 1-based number of the first line that is not a triple
+ */
+export function parseNTriples(text: string): { triples: Triple[]; badLine: number | null } {
+    const triples: Triple[] = [];
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmed = line.trim();
+        if (trimmed === "" || trimmed.startsWith("#")) {
+            continue;
+        }
+        const s = readTerm(line, 0);
+        const p = s === null ? null : readTerm(line, s.end);
+        const o = p === null ? null : readTerm(line, p.end);
+        END.lastIndex = o?.end ?? 0;
+        if (
+            s === null ||
+            p === null ||
+            o === null ||
+            s.term.kind === "literal" ||
+            p.term.kind !== "iri" ||
+            !END.test(line)
+        ) {
+            return { triples, badLine: i + 1 };
+        }
+        triples.push({ s: s.term, p: p.term.value, o: o.term });
+    }
+    return { triples, badLine: null };
+}
+
+/**
+ * The local name of an IRI: what follows its last `#`, else its last `/`, else its last `:`.
+ * @param iri - the IRI
+ * @returns the local name, or the whole IRI when nothing follows
+ */
+export function localName(iri: string): string {
+    const cut = Math.max(iri.lastIndexOf("#"), iri.lastIndexOf("/"), iri.lastIndexOf(":"));
+    const name = iri.slice(cut + 1);
+    return name === "" ? iri : name;
+}
+
+/**
+ * A literal's value: numbers and booleans as such, everything else as its text.
+ * @param term - the literal
+ * @returns the value
+ */
+function literalValue(term: Extract<Term, { kind: "literal" }>): string | number | boolean {
+    if (NUMERIC.has(term.datatype)) {
+        const n = Number(term.value);
+        return Number.isFinite(n) ? n : term.value;
+    }
+    if (term.datatype === `${XSD}boolean`) {
+        return term.value === "true" || term.value === "1";
+    }
+    return term.value;
+}
+
+/**
+ * Whether a literal's language tag passes a language filter (SPARQL's langMatches, plus untagged).
+ * @param lang - the literal's tag, lowercase
+ * @param want - the filter, lowercase, or "" for none
+ * @returns whether to keep it
+ */
+function languageMatches(lang: string, want: string): boolean {
+    return want === "" || lang === "" || lang === want || lang.startsWith(`${want}-`);
+}
+
+/** The nodes and edges of some triples. */
+export interface Graphed {
+    /** Each subject's literal attributes, by subject id; a predicate's first value wins. */
+    readonly nodes: Map<string, Record<string, string | number | boolean>>;
+    /** One edge per IRI or blank-node object. */
+    readonly edges: { source: string; target: string; type: string; predicate: string }[];
+}
+
+/**
+ * Turn triples into nodes and edges.
+ * @param triples - the triples
+ * @param language - the language filter, lowercase, or ""
+ * @param into - nodes and edges to add to
+ * @returns the nodes and edges
+ */
+export function graphTriples(triples: readonly Triple[], language: string, into?: Graphed): Graphed {
+    const out: Graphed = into ?? { nodes: new Map(), edges: [] };
+    for (const { s, p, o } of triples) {
+        let attrs = out.nodes.get(s.value);
+        if (attrs === undefined) {
+            attrs = {};
+            out.nodes.set(s.value, attrs);
+        }
+        if (o.kind === "literal") {
+            const key = localName(p);
+            // ponytail: one value per predicate (the first); a list column if multi-valued literals matter
+            if (languageMatches(o.lang, language) && !(key in attrs)) {
+                attrs[key] = literalValue(o);
+            }
+        } else {
+            out.edges.push({ source: s.value, target: o.value, type: localName(p), predicate: p });
+        }
+    }
+    return out;
+}
+
+/**
+ * One canonical line per triple, for the content hash.
+ * @param t - the triple
+ * @returns the line
+ */
+export function canonicalTriple(t: Triple): string {
+    const o = t.o.kind === "literal" ? JSON.stringify([t.o.value, t.o.lang, t.o.datatype]) : t.o.value;
+    return `${t.s.value} ${t.p} ${o}`;
+}

package/src/sources/sparql/params.ts

@@ -0,0 +1,28 @@
+/**
+ * The parameters of the SPARQL source. Experimental.
+ */
+
+import { type NodeId } from "@graphty/graph-format";
+
+/**
+ * What to read from a SPARQL endpoint. Experimental.
+ * @category Network sources
+ */
+export interface SparqlParams {
+    /** The caller's CONSTRUCT or DESCRIBE query; without one a query is generated from `mode`. */
+    readonly query?: string | undefined;
+    /** "query" (the caller's), "all" (every triple, paged) or "neighborhood" (the triples of `ids`). */
+    readonly mode?: "query" | "all" | "neighborhood" | undefined;
+    /** The seed IRIs of "neighborhood". */
+    readonly ids?: readonly NodeId[] | undefined;
+    /** Keep literals tagged with this language (`en`) or with none; default every literal. */
+    readonly language?: string | undefined;
+    /**
+     * Generated queries and expansions follow only predicates starting with one of these IRIs
+     * (Wikidata's direct claims: `http://www.wikidata.org/prop/direct/`); literals are always kept.
+     * Default every predicate.
+     */
+    readonly predicates?: readonly string[] | string | undefined;
+    /** Solutions per page, 1 to 10000. A load stops between pages. @defaultValue 1000 */
+    readonly pageSize?: number | undefined;
+}