@graphty/graph-io 0.3.48 → 0.3.49

package/src/sources/sparql/ntriples.ts

@@ -0,0 +1,219 @@
+/**
+ * Reading the N-Triples a CONSTRUCT or DESCRIBE answers, and turning triples into nodes and edges:
+ * an IRI or blank-node object is an edge from the subject, a literal is an attribute of the
+ * subject named by the predicate's local name.
+ */
+
+/** One RDF term. */
+export type Term =
+    | { readonly kind: "iri"; readonly value: string }
+    | { readonly kind: "bnode"; readonly value: string }
+    | { readonly kind: "literal"; readonly value: string; readonly lang: string; readonly datatype: string };
+
+/** One triple. */
+export interface Triple {
+    /** The subject: an IRI or a blank node. */
+    readonly s: Term;
+    /** The predicate IRI. */
+    readonly p: string;
+    /** The object. */
+    readonly o: Term;
+}
+
+/** One term at the sticky position: an IRI, a blank node, or a literal with its tag or type. */
+const TERM = /\s*(?:<([^<>]*)>|_:([\w.-]*[\w-])|"((?:[^"\\]|\\.)*)"(?:@([A-Za-z][\w-]*)|\^\^<([^<>]*)>)?)/y;
+
+/** The end of a statement: a dot, then an optional comment. */
+const END = /\s*\.\s*(?:#.*)?$/y;
+
+/** The XSD namespace. */
+const XSD = "http://www.w3.org/2001/XMLSchema#";
+
+/** The XSD numeric types, read as numbers. */
+const NUMERIC: ReadonlySet<string> = new Set(
+    [
+        "integer",
+        "decimal",
+        "double",
+        "float",
+        "int",
+        "long",
+        "short",
+        "byte",
+        "nonNegativeInteger",
+        "nonPositiveInteger",
+        "positiveInteger",
+        "negativeInteger",
+        "unsignedInt",
+        "unsignedLong",
+        "unsignedShort",
+        "unsignedByte",
+    ].map((t) => XSD + t),
+);
+
+/**
+ * Undo N-Triples escapes.
+ * @param text - escaped text
+ * @returns the text
+ */
+function unescape(text: string): string {
+    if (!text.includes("\\")) {
+        return text;
+    }
+    const simple: Readonly<Record<string, string>> = { t: "\t", b: "\b", n: "\n", r: "\r", f: "\f" };
+    return text.replaceAll(
+        /\\(?:u([\da-fA-F]{4})|U([\da-fA-F]{8})|(.))/g,
+        (_m, u4?: string, u8?: string, c?: string) => {
+            const hex = u4 ?? u8;
+            if (hex !== undefined) {
+                return String.fromCodePoint(Number.parseInt(hex, 16));
+            }
+            return simple[c ?? ""] ?? c ?? "";
+        },
+    );
+}
+
+/**
+ * Read one term at `at`.
+ * @param line - the line
+ * @param at - where to start
+ * @returns the term and where it ended, or null when there is none
+ */
+function readTerm(line: string, at: number): { term: Term; end: number } | null {
+    TERM.lastIndex = at;
+    const m = TERM.exec(line);
+    if (m === null) {
+        return null;
+    }
+    const [, iri, bnode, lexical, lang, datatype] = m;
+    let term: Term;
+    if (iri !== undefined) {
+        term = { kind: "iri", value: unescape(iri) };
+    } else if (bnode === undefined) {
+        term = {
+            kind: "literal",
+            value: unescape(lexical ?? ""),
+            lang: lang?.toLowerCase() ?? "",
+            datatype: datatype ?? "",
+        };
+    } else {
+        term = { kind: "bnode", value: `_:${bnode}` };
+    }
+    return { term, end: TERM.lastIndex };
+}
+
+/**
+ * Parse N-Triples.
+ * @param text - the document
+ * @returns the triples, or the 1-based number of the first line that is not a triple
+ */
+export function parseNTriples(text: string): { triples: Triple[]; badLine: number | null } {
+    const triples: Triple[] = [];
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmed = line.trim();
+        if (trimmed === "" || trimmed.startsWith("#")) {
+            continue;
+        }
+        const s = readTerm(line, 0);
+        const p = s === null ? null : readTerm(line, s.end);
+        const o = p === null ? null : readTerm(line, p.end);
+        END.lastIndex = o?.end ?? 0;
+        if (
+            s === null ||
+            p === null ||
+            o === null ||
+            s.term.kind === "literal" ||
+            p.term.kind !== "iri" ||
+            !END.test(line)
+        ) {
+            return { triples, badLine: i + 1 };
+        }
+        triples.push({ s: s.term, p: p.term.value, o: o.term });
+    }
+    return { triples, badLine: null };
+}
+
+/**
+ * The local name of an IRI: what follows its last `#`, else its last `/`, else its last `:`.
+ * @param iri - the IRI
+ * @returns the local name, or the whole IRI when nothing follows
+ */
+export function localName(iri: string): string {
+    const cut = Math.max(iri.lastIndexOf("#"), iri.lastIndexOf("/"), iri.lastIndexOf(":"));
+    const name = iri.slice(cut + 1);
+    return name === "" ? iri : name;
+}
+
+/**
+ * A literal's value: numbers and booleans as such, everything else as its text.
+ * @param term - the literal
+ * @returns the value
+ */
+function literalValue(term: Extract<Term, { kind: "literal" }>): string | number | boolean {
+    if (NUMERIC.has(term.datatype)) {
+        const n = Number(term.value);
+        return Number.isFinite(n) ? n : term.value;
+    }
+    if (term.datatype === `${XSD}boolean`) {
+        return term.value === "true" || term.value === "1";
+    }
+    return term.value;
+}
+
+/**
+ * Whether a literal's language tag passes a language filter (SPARQL's langMatches, plus untagged).
+ * @param lang - the literal's tag, lowercase
+ * @param want - the filter, lowercase, or "" for none
+ * @returns whether to keep it
+ */
+function languageMatches(lang: string, want: string): boolean {
+    return want === "" || lang === "" || lang === want || lang.startsWith(`${want}-`);
+}
+
+/** The nodes and edges of some triples. */
+export interface Graphed {
+    /** Each subject's literal attributes, by subject id; a predicate's first value wins. */
+    readonly nodes: Map<string, Record<string, string | number | boolean>>;
+    /** One edge per IRI or blank-node object. */
+    readonly edges: { source: string; target: string; type: string; predicate: string }[];
+}
+
+/**
+ * Turn triples into nodes and edges.
+ * @param triples - the triples
+ * @param language - the language filter, lowercase, or ""
+ * @param into - nodes and edges to add to
+ * @returns the nodes and edges
+ */
+export function graphTriples(triples: readonly Triple[], language: string, into?: Graphed): Graphed {
+    const out: Graphed = into ?? { nodes: new Map(), edges: [] };
+    for (const { s, p, o } of triples) {
+        let attrs = out.nodes.get(s.value);
+        if (attrs === undefined) {
+            attrs = {};
+            out.nodes.set(s.value, attrs);
+        }
+        if (o.kind === "literal") {
+            const key = localName(p);
+            // ponytail: one value per predicate (the first); a list column if multi-valued literals matter
+            if (languageMatches(o.lang, language) && !(key in attrs)) {
+                attrs[key] = literalValue(o);
+            }
+        } else {
+            out.edges.push({ source: s.value, target: o.value, type: localName(p), predicate: p });
+        }
+    }
+    return out;
+}
+
+/**
+ * One canonical line per triple, for the content hash.
+ * @param t - the triple
+ * @returns the line
+ */
+export function canonicalTriple(t: Triple): string {
+    const o = t.o.kind === "literal" ? JSON.stringify([t.o.value, t.o.lang, t.o.datatype]) : t.o.value;
+    return `${t.s.value} ${t.p} ${o}`;
+}

package/src/sources/sparql/params.ts

@@ -0,0 +1,28 @@
+/**
+ * The parameters of the SPARQL source. Experimental.
+ */
+
+import { type NodeId } from "@graphty/graph-format";
+
+/**
+ * What to read from a SPARQL endpoint. Experimental.
+ * @category Network sources
+ */
+export interface SparqlParams {
+    /** The caller's CONSTRUCT or DESCRIBE query; without one a query is generated from `mode`. */
+    readonly query?: string | undefined;
+    /** "query" (the caller's), "all" (every triple, paged) or "neighborhood" (the triples of `ids`). */
+    readonly mode?: "query" | "all" | "neighborhood" | undefined;
+    /** The seed IRIs of "neighborhood". */
+    readonly ids?: readonly NodeId[] | undefined;
+    /** Keep literals tagged with this language (`en`) or with none; default every literal. */
+    readonly language?: string | undefined;
+    /**
+     * Generated queries and expansions follow only predicates starting with one of these IRIs
+     * (Wikidata's direct claims: `http://www.wikidata.org/prop/direct/`); literals are always kept.
+     * Default every predicate.
+     */
+    readonly predicates?: readonly string[] | string | undefined;
+    /** Solutions per page, 1 to 10000. A load stops between pages. @defaultValue 1000 */
+    readonly pageSize?: number | undefined;
+}

package/src/sources/sparql/query.ts

@@ -0,0 +1,162 @@
+/**
+ * The SPARQL the source sends: the caller's CONSTRUCT or DESCRIBE with the paging the source
+ * adds (`ORDER BY` the template's variables, `LIMIT` and `OFFSET`), and the generated queries of
+ * "all", "neighborhood" and expand (`VALUES ?s { ... }`).
+ */
+
+/** Characters an IRIREF may not hold. */
+const NOT_IRI = /[\s<>"{}|^`\\]/;
+
+/** A language tag a filter may name. */
+const LANGUAGE = /^[a-z]{1,8}(-[a-z\d]{1,8})*$/i;
+
+/** The query forms, as the first keyword after the prologue. */
+type QueryForm = "construct" | "describe" | "select" | "ask" | null;
+
+/** What paging the caller's query takes. */
+interface UserQuery {
+    /** Its form. */
+    readonly form: QueryForm;
+    /** Whether it sets its own LIMIT or OFFSET, so the source sends it as written. */
+    readonly ownLimit: boolean;
+    /** Whether it orders its solutions itself. */
+    readonly ordered: boolean;
+    /** The variables of a CONSTRUCT template, in order, for ORDER BY. */
+    readonly templateVars: readonly string[];
+}
+
+/**
+ * The query with IRIs, strings and comments blanked out, so keywords and braces can be found.
+ * @param query - the query
+ * @returns the bare text
+ */
+function bare(query: string): string {
+    return query
+        .replaceAll(/<[^<>\s]*>/g, "<>")
+        .replaceAll(/"""[\s\S]*?"""|'''[\s\S]*?'''|"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'/g, '""')
+        .replaceAll(/#[^\n]*/g, "");
+}
+
+/**
+ * Read what paging a caller's query takes.
+ * @param query - the query
+ * @returns its form, its own modifiers and its template variables
+ */
+export function readQuery(query: string): UserQuery {
+    const text = bare(query);
+    const form = /\b(CONSTRUCT|DESCRIBE|SELECT|ASK)\b/i.exec(text)?.[1]?.toLowerCase() ?? null;
+    // the solution modifiers follow the last closing brace (a trailing VALUES block is not handled)
+    const tail = text.slice(text.lastIndexOf("}") + 1);
+    let templateVars: string[] = [];
+    if (form === "construct") {
+        const template = /\bCONSTRUCT\s*\{([^}]*)\}/i.exec(text)?.[1] ?? "";
+        templateVars = [...new Set(template.match(/[?$]\w+/g) ?? [])].map((v) => `?${v.slice(1)}`);
+    }
+    return {
+        form: form as QueryForm,
+        ownLimit: /\b(LIMIT|OFFSET)\b/i.test(tail),
+        ordered: /\bORDER\s+BY\b/i.test(tail),
+        templateVars,
+    };
+}
+
+/**
+ * The caller's query with one page's modifiers added.
+ * @param query - the query
+ * @param shape - what readQuery found
+ * @param limit - solutions per page
+ * @param offset - solutions to skip
+ * @returns the query to send
+ */
+export function pagedQuery(query: string, shape: UserQuery, limit: number, offset: number): string {
+    const order = shape.ordered || shape.templateVars.length === 0 ? "" : `\nORDER BY ${shape.templateVars.join(" ")}`;
+    const skip = offset > 0 ? ` OFFSET ${String(offset)}` : "";
+    return `${query.trimEnd()}${order}\nLIMIT ${String(limit)}${skip}`;
+}
+
+/**
+ * An IRI as a SPARQL term, or null when it cannot be one.
+ * @param id - the id
+ * @returns `<iri>`, or null
+ */
+export function iriTerm(id: unknown): string | null {
+    const text = String(id);
+    return /^[a-z][\w+.-]*:/i.test(text) && !NOT_IRI.test(text) ? `<${text}>` : null;
+}
+
+/** What generated queries keep: literals in a language, edges along some predicates. */
+interface TripleFilter {
+    /** Keep literals tagged with this language or untagged; "" keeps every literal. */
+    readonly language: string;
+    /** Keep edges whose predicate starts with one of these IRIs; empty keeps every edge. */
+    readonly predicates: readonly string[];
+}
+
+/**
+ * The FILTERs of generated queries, or "" for none. Both values are checked by the caller.
+ * @param filter - the language and the predicate prefixes
+ * @returns the FILTER clauses
+ */
+function filters(filter: TripleFilter): string {
+    const { language, predicates } = filter;
+    const lang =
+        language === "" ? "" : ` FILTER(!isLiteral(?o) || lang(?o) = "" || langMatches(lang(?o), "${language}"))`;
+    const along = predicates.map((p) => `STRSTARTS(STR(?p), "${p}")`).join(" || ");
+    return predicates.length === 0 ? lang : `${lang} FILTER(isLiteral(?o) || ${along})`;
+}
+
+/**
+ * Whether a language tag may go into a FILTER.
+ * @param language - the tag
+ * @returns whether it is well formed
+ */
+export function validLanguage(language: string): boolean {
+    return LANGUAGE.test(language);
+}
+
+/**
+ * Mode "all": every triple, ordered so pages are stable.
+ * @param filter - what to keep
+ * @param limit - solutions per page
+ * @param offset - solutions to skip
+ * @returns the query
+ */
+export function allQuery(filter: TripleFilter, limit: number, offset: number): string {
+    return pagedQuery(
+        `CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o .${filters(filter)} }`,
+        { form: "construct", ownLimit: false, ordered: false, templateVars: ["?s", "?p", "?o"] },
+        limit,
+        offset,
+    );
+}
+
+/**
+ * The triples around some IRIs: out (`VALUES ?s`), in (`VALUES ?o`) or both.
+ * @param iris - the IRIs as terms
+ * @param direction - which triples
+ * @param filter - what to keep
+ * @param limit - solutions per page
+ * @param offset - solutions to skip
+ * @returns the query
+ */
+export function aroundQuery(
+    iris: readonly string[],
+    direction: "out" | "in" | "both",
+    filter: TripleFilter,
+    limit: number,
+    offset: number,
+): string {
+    const values = iris.join(" ");
+    const out = `{ VALUES ?s { ${values} } ?s ?p ?o .${filters(filter)} }`;
+    const inward = `{ VALUES ?o { ${values} } ?s ?p ?o .${filters({ ...filter, language: "" })} }`;
+    let where = `${out} UNION ${inward}`;
+    if (direction !== "both") {
+        where = direction === "out" ? out : inward;
+    }
+    return pagedQuery(
+        `CONSTRUCT { ?s ?p ?o } WHERE { ${where} }`,
+        { form: "construct", ownLimit: false, ordered: false, templateVars: ["?s", "?p", "?o"] },
+        limit,
+        offset,
+    );
+}

package/src/sources/types.ts

@@ -223,6 +223,18 @@ export interface RetryOptions {
     readonly maxDelayMs?: number | undefined;
 }
 
+/**
+ * How much a source holds for a load, as it counted it, so a caller can show "100 of 28,455".
+ * Experimental.
+ * @category Network sources
+ */
+export interface SourceTotal {
+    /** The nodes the load's params match at the source. */
+    readonly nodes?: number | undefined;
+    /** The edges the load's params match at the source. */
+    readonly edges?: number | undefined;
+}
+
 /**
  * Facts a source reports with `ctx.finish()`. Experimental.
  * @category Network sources
@@ -243,6 +255,8 @@ export interface SourceFinish {
         | undefined;
     /** What lets a later call tell whether the data changed. */
     readonly validators?: SourceValidators | undefined;
+    /** What the source holds for this load, when it counted: the nodes and edges it matches. */
+    readonly total?: SourceTotal | undefined;
 }
 
 /**
@@ -357,6 +371,11 @@ export interface SourceSample {
         | undefined;
     /** false: too big to load whole; start from a neighborhood. */
     readonly whole?: boolean | undefined;
+    /**
+     * A node cap for loading the sample when the caller gives none, for a sample too big to load
+     * whole that pages: the load stops at the first page past it, and its cursor reads on.
+     */
+    readonly firstPage?: number | undefined;
 }
 
 /**
@@ -377,6 +396,20 @@ export interface SourceDescriptor {
     readonly auth?: readonly CredentialKind[] | undefined;
     /** Whether a credential is needed. @defaultValue "none" */
     readonly authRequired?: "required" | "optional" | "none" | undefined;
+    /**
+     * Where an `api-key` credential goes, for a form that asks for the key alone: pass it to
+     * `credentials.apiKey(descriptor.apiKey, key)`. Absent when the caller must say.
+     */
+    readonly apiKey?:
+        | {
+              /** The request header that carries the key. */
+              readonly header: string;
+          }
+        | {
+              /** The query parameter that carries the key. */
+              readonly query: string;
+          }
+        | undefined;
     /** The endpoint used when the caller names none. @defaultValue null */
     readonly defaultEndpoint?: string | null | undefined;
     /** Where it runs; a runtime marked unsupported is refused with E_SOURCE_RUNTIME before any request. */
@@ -584,6 +617,8 @@ export interface SourceReportInfo {
         | undefined;
     /** What lets a later call tell whether the data changed. */
     readonly validators?: SourceValidators | undefined;
+    /** What the source holds for this load, when it counted: the nodes and edges it matches. */
+    readonly total?: SourceTotal | undefined;
 }
 
 /**