@graphty/graph-io 0.3.49 → 0.3.50

package/src/sources/credentials.ts

@@ -7,7 +7,13 @@
  */
 
 import { failAlone } from "./fail.js";
-import { type AwsCredentials, type Credential, type CredentialKind } from "./types.js";
+import {
+    type AwsCredentials,
+    type Credential,
+    type CredentialKind,
+    type SourceCredentialSlot,
+    type SourceDescriptor,
+} from "./types.js";
 
 /**
  * Where a credential may go: the load's endpoint origin by default. Experimental.
@@ -145,6 +151,63 @@ function base64(text: string): string {
     return btoa(binary);
 }
 
+/**
+ * The plain value a slot's credential is made from: a key or a token, or a user name and password.
+ * @category Network sources
+ */
+export type SlotValue =
+    | string
+    | {
+          /** A basic slot's user name. */
+          readonly username: string;
+          /** A basic slot's password. */
+          readonly password: string;
+      };
+
+/**
+ * The credential a slot describes, from its plain value.
+ * @param slot - the slot
+ * @param value - the key or token, or a basic slot's user name and password
+ * @param bind - the origins it may be sent to
+ * @returns the credential
+ */
+function forSlot(slot: SourceCredentialSlot, value: SlotValue, bind?: CredentialBind): Credential {
+    if (typeof value !== "string") {
+        if (slot.kind === "basic") {
+            return new SourceCredential({ kind: "basic", username: value.username, password: value.password }, bind);
+        }
+    } else if (slot.kind === "bearer") {
+        return new SourceCredential({ kind: "bearer", token: value }, bind);
+    } else if (slot.kind === "api-key") {
+        const send = slot.send ?? { query: slot.name };
+        const header = "header" in send ? send.header : null;
+        const query = "query" in send ? send.query : null;
+        return new SourceCredential({ kind: "api-key", header, query, value }, bind);
+    }
+    return failAlone("E_SOURCE_PARAM", `a ${slot.kind} slot cannot take this value`, {
+        name: "credential",
+        reason: "type",
+    });
+}
+
+/**
+ * The credential slots a source takes: the ones its descriptor declares, else one slot `default`
+ * of its first credential kind when it takes a credential (`auth`), else none. Experimental.
+ * @param descriptor - the source's descriptor
+ * @returns the slots
+ * @category Network sources
+ */
+export function credentialSlots(descriptor: SourceDescriptor | undefined): readonly SourceCredentialSlot[] {
+    if (descriptor?.credentials !== undefined) {
+        return descriptor.credentials;
+    }
+    const kind = descriptor?.auth?.[0];
+    if (kind === undefined || descriptor?.authRequired === "none") {
+        return [];
+    }
+    return [{ name: "default", kind, required: descriptor?.authRequired === "required" }];
+}
+
 /**
  * The factories that make credentials. Pass the result as the `credential` option of a load; it is
  * sent only to the load's endpoint origin, or to `bind.origins` when you give them. Experimental.
@@ -213,6 +276,21 @@ export const credentials = Object.freeze({
     custom(apply: (request: Request) => Promise<Request> | Request, kind: string, bind?: CredentialBind): Credential {
         return new SourceCredential({ kind: "custom", name: kind, apply }, bind);
     },
+    /**
+     * The credential a source's slot describes (`credentialSlots(descriptor)`), from its plain
+     * value: an API key sent as the slot says, a bearer token, or a basic slot's user name and
+     * password.
+     * @example
+     * ```ts
+     * const [slot] = credentialSlots(openalex.descriptor); // api_key, sent as ?api_key=
+     * const credential = credentials.forSlot(slot, key);
+     * ```
+     * @param slot - the slot
+     * @param value - the key or token, or `{ username, password }` for a basic slot
+     * @param bind - the origins it may be sent to
+     * @returns the credential
+     */
+    forSlot,
 });
 
 /**
@@ -248,6 +326,17 @@ export function credentialHeader(credential: Credential): string | null {
         : null;
 }
 
+/**
+ * The query parameter a credential is sent in, so a cursor can leave it out; null for one sent in
+ * a header or not at all.
+ * @param credential - the credential
+ * @returns the parameter's name
+ */
+export function credentialQuery(credential: Credential): string | null {
+    const secret = secretOf(credential);
+    return secret?.kind === "api-key" && secret.header === null ? (secret.query ?? "api_key") : null;
+}
+
 /**
  * Add a credential to a request. Internal to graph-io: only `ctx.fetch` calls it.
  * @param credential - the credential

package/src/sources/cypher/index.ts

@@ -253,7 +253,8 @@ class Reader {
      * Mode "query": the reader's own query, as written. With a node cap (or a cursor from a capped
      * run) the server returns at most that many rows (`CALL { query } RETURN * LIMIT n`), and a full
      * result hands back a cursor that runs the query again with twice the rows, for the caller to
-     * merge. A server that refuses the wrapper runs the query uncapped.
+     * merge. A server that refuses the wrapper fails the load with `E_SOURCE_PARAM` (`name:
+     * "maxNodes"`, `reason: "uncappable"`): the query is never run past the cap the caller set.
      * @param query - the query
      * @param cursor - where to resume: `{ rows }` from an earlier capped run
      * @returns the one page
@@ -267,7 +268,6 @@ class Reader {
                 ? null
                 : Number(checkedInteger(asked, "maxNodes", 1, Number.MAX_SAFE_INTEGER, ctx.report));
         let result: QueryResult;
-        let capped = rows !== null;
         try {
             result = await this.transport.run(
                 rows === null
@@ -276,16 +276,18 @@ class Reader {
                 parameters,
             );
         } catch (err) {
-            if (!capped || !(err instanceof ImportError) || err.issue?.code !== SOURCE_ISSUE.E_SOURCE_QUERY) {
+            if (rows === null || !(err instanceof ImportError) || err.issue?.code !== SOURCE_ISSUE.E_SOURCE_QUERY) {
                 throw err;
             }
-            capped = false;
-            result = await this.transport.run(query, parameters);
+            return failOn(ctx.report, "E_SOURCE_PARAM", "the server cannot cap this query; load it without maxNodes", {
+                name: "maxNodes",
+                reason: "uncappable",
+            });
         }
         const collected = new Collector();
         collected.add(result, false);
         requireGraph(result, collected, ctx.report);
-        const full = capped && rows !== null && result.rows.length >= rows;
+        const full = rows !== null && result.rows.length >= rows;
         return { collected, next: full ? { rows: rows * 2 } : null };
     }
 
@@ -362,13 +364,12 @@ class Reader {
                 (d) => nodePage(profile, this.filters, String(size), after !== null, d),
                 after === null ? {} : { after },
             );
-            const last = rows.at(-1)?.[fields.indexOf("key")];
+            const at = fields.indexOf("key");
+            const last = rows.at(-1)?.[at];
             const key = typeof last === "string" || typeof last === "number" ? last : null;
             if (key !== null) {
-                const among = await this.transport.run(edgesAmong(profile, this.filters, after !== null), {
-                    last: key,
-                    ...(after === null ? {} : { after }),
-                });
+                const keys = rows.map((row) => row[at]);
+                const among = await this.transport.run(edgesAmong(profile, this.filters), { keys, last: key });
                 collected.add(among, false);
             }
             const next = rows.length >= size && key !== null ? { phase: "nodes", after: key } : null;

package/src/sources/cypher/query-builder.ts

@@ -216,25 +216,25 @@ export function edgePage(profile: CypherProfile, filters: QueryFilters, pageSize
 }
 
 /**
- * The relationships among the nodes a capped mode "all" has loaded: both ends at or before the
- * node page's last key `$last`, and, after the first page, one end past the previous page's last
- * key `$after`, so each relationship arrives with the page that loads its later end.
+ * The relationships among the nodes a capped mode "all" has loaded that touch one node page: each
+ * starts from the page's own nodes `$keys` (an id or index lookup, never a scan of every
+ * relationship), with its other end on this page or an earlier one (at or before `$last`), so each
+ * relationship arrives once, with the page that loads its later end.
  * @param profile - the server's profile
  * @param filters - the filters
- * @param after - whether earlier pages were loaded
- * @returns the statement; it reads `$last` and `$after`
+ * @returns the statement; it reads `$keys` and `$last`
  */
-export function edgesAmong(profile: CypherProfile, filters: QueryFilters, after: boolean): string {
-    const where = [nodePredicate("a", filters), nodePredicate("b", filters)].filter((p) => p !== null);
+export function edgesAmong(profile: CypherProfile, filters: QueryFilters): string {
+    const ka = idExpression("a", profile, filters);
+    const kb = idExpression("b", profile, filters);
+    const ends = [nodePredicate("a", filters), nodePredicate("b", filters)].filter((p) => p !== null);
+    const rest = ends.length === 0 ? "" : ` AND ${ends.join(" AND ")}`;
+    const pattern = `MATCH (a)-[r${typesPattern(filters)}]->(b)`;
     return [
-        `MATCH (a)-[r${typesPattern(filters)}]->(b)`,
-        where.length === 0 ? null : `WHERE ${where.join(" AND ")}`,
-        `WITH a, r, b, ${idExpression("a", profile, filters)} AS ka, ${idExpression("b", profile, filters)} AS kb`,
-        `WHERE ka <= $last AND kb <= $last${after ? " AND (ka > $after OR kb > $after)" : ""}`,
-        "RETURN a, r, b",
-    ]
-        .filter((l) => l !== null)
-        .join(" ");
+        `${pattern} WHERE ${ka} IN $keys AND ${kb} <= $last${rest} RETURN a, r, b`,
+        "UNION ALL",
+        `${pattern} WHERE ${kb} IN $keys AND ${ka} <= $last AND NOT ${ka} IN $keys${rest} RETURN a, r, b`,
+    ].join(" ");
 }
 
 /** How far a neighborhood reaches. */

package/src/sources/rest/descriptor.ts

@@ -49,6 +49,10 @@ export const OPENALEX_DESCRIPTOR: SourceDescriptor = Object.freeze({
     defaultEndpoint: "https://api.openalex.org",
     auth: ["api-key"],
     authRequired: "optional",
+    // OpenAlex documents its key as the api_key query parameter
+    credentials: [
+        { name: "api_key", kind: "api-key", required: false, plainName: "API key", send: { query: "api_key" } },
+    ],
     apiKey: { query: "api_key" },
     rateLimit: { requests: 10, perMs: 1000 },
     features: ["expand", "probe", "validate", "page"],

package/src/sources/rest/openalex.ts

@@ -266,8 +266,10 @@ const methods = restMethods<OpenAlexParams>(plan, false);
  * The OpenAlex preset (https://openalex.org): `work` (the citation graph around a work, default)
  * or `author` (the works they wrote), `graph: "coauthorship"` for authors joined by shared works,
  * and `include` (`"authors"`, `"institutions"`, `"topics"`) for more node kinds. Without a key,
- * the first page loads and a later one warns W_SOURCE_AUTH with `report.source.next`; pass
- * `credentials.apiKey({ query: "api_key" }, key)` and continue with the cursor. Experimental.
+ * the first page loads and a later one warns W_SOURCE_AUTH with `report.source.next`. Its one
+ * optional credential slot, `api_key`, is sent as the `api_key` query parameter: pass
+ * `credentials.forSlot(slot, key)` (or `credentials.apiKey({ query: "api_key" }, key)`) and
+ * continue with the cursor. Experimental.
  * @category Network sources
  */
 export const openalexSource: GraphSource<OpenAlexParams> = defineSource<OpenAlexParams>({

package/src/sources/rest/source.ts

@@ -19,6 +19,7 @@ import {
     type GraphSource,
     type RecordBatch,
     type SourceContext,
+    type SourceCredentialSlot,
     type SourceCursor,
     type SourceDescriptor,
     type SourceFinish,
@@ -697,6 +698,29 @@ export type RestSourceInfo = {
     readonly id: string;
 } & Partial<SourceDescriptor>;
 
+/**
+ * The credential slot a mapping's `auth` describes: an API key's slot is named by the query
+ * parameter or header it is sent in (`{ query: "api_key" }` is the slot `api_key`), a bearer
+ * token's is `default`. Optional, as a mapping cannot say a key is required.
+ * @param mapping - the mapping
+ * @returns the slots
+ */
+function mappingSlots(mapping: RestMapping): readonly SourceCredentialSlot[] {
+    const { auth } = mapping;
+    if (auth === undefined) {
+        return [];
+    }
+    if (auth.kind === "bearer") {
+        return [{ name: "default", kind: "bearer", required: false }];
+    }
+    const { header } = auth;
+    if (header !== undefined) {
+        return [{ name: header.toLowerCase(), kind: "api-key", required: false, send: { header } }];
+    }
+    const query = auth.query ?? "api_key";
+    return [{ name: query, kind: "api-key", required: false, send: { query } }];
+}
+
 /**
  * Where a mapping's API key goes, as a descriptor says it.
  * @param auth - the mapping's auth
@@ -748,6 +772,7 @@ export function restSource(mapping: RestMapping, info: RestSourceInfo): GraphSou
         defaultEndpoint: mapping.endpoint,
         auth,
         authRequired: auth.length === 0 ? "none" : "optional",
+        credentials: mappingSlots(mapping),
         ...apiKeyOf(mapping.auth),
         ...(mapping.rateLimit === undefined ? {} : { rateLimit: mapping.rateLimit }),
         ...fields,

package/src/sources/run.ts

@@ -15,7 +15,7 @@ import { type FormatRegistry } from "../registry.js";
 import { SNIFF_HEAD_BYTES } from "../sniff.js";
 import { type CommonImportOptions, ImportError, type ImportInput, type ImportReport } from "../types.js";
 import { type CreatedContext, createSourceContext, detectRuntime, LimitStop } from "./context.js";
-import { originOf } from "./credentials.js";
+import { credentialQuery, originOf } from "./credentials.js";
 import { checkParams } from "./define.js";
 import { failOn, warnOn } from "./fail.js";
 import { SOURCE_FORMAT_PREFIX } from "./ids.js";
@@ -60,6 +60,8 @@ interface Prepared {
     readonly params: Record<string, unknown>;
     /** The sink wrapper's counters, set once the sink is known. */
     readonly counts: { nodes: Set<number>; edgesAtStart: number; sink: GraphSink | null };
+    /** The query parameter the call's credential is sent in, which a cursor never keeps; null for none. */
+    readonly secretQuery: string | null;
 }
 
 /**
@@ -131,7 +133,8 @@ function prepare(
             edges: counts.sink === null ? 0 : counts.sink.edgeCount - counts.edgesAtStart,
         }),
     });
-    return { report, created, params, counts };
+    const secretQuery = options.credential === undefined ? null : credentialQuery(options.credential);
+    return { report, created, params, counts, secretQuery };
 }
 
 /**
@@ -721,6 +724,7 @@ export async function runMethod<T>(
  */
 export function withSource(report: ImportReport, prepared: Prepared, next: SourceCursor | null): ImportReport {
     const own = prepared.report.finish();
+    const cursor = prepared.secretQuery === null || next === null ? next : withoutParam(next, prepared.secretQuery);
     // issues the context recorded (retries, limits) that a source's own report does not hold
     const missing = own.issues.filter((i) => !report.issues.includes(i));
     return Object.freeze({
@@ -728,10 +732,38 @@ export function withSource(report: ImportReport, prepared: Prepared, next: Sourc
         issues: missing.length === 0 ? report.issues : Object.freeze([...report.issues, ...missing]),
         warningCount: report.warningCount + missing.filter((i) => i.severity === "warning").length,
         errorCount: report.errorCount + missing.filter((i) => i.severity === "error").length,
-        source: prepared.created.info(next),
+        source: prepared.created.info(cursor),
     });
 }
 
+/**
+ * A cursor without a query parameter in any URL it holds: a next-page link a service returns
+ * repeats the API key the request carried, and a cursor is saved where a credential never is.
+ * The credential is sent again with every request, so the next page does not need it there.
+ * @param value - the cursor, or a value inside it
+ * @param name - the parameter
+ * @returns the value without it
+ */
+function withoutParam<T>(value: T, name: string): T {
+    if (typeof value === "string" && /^https?:\/\//i.test(value)) {
+        const url = new URL(value);
+        if (!url.searchParams.has(name)) {
+            return value;
+        }
+        url.searchParams.delete(name);
+        return url.href as T;
+    }
+    if (Array.isArray(value)) {
+        return Object.freeze(value.map((each: unknown) => withoutParam(each, name))) as T;
+    }
+    if (typeof value === "object" && value !== null) {
+        return Object.freeze(
+            Object.fromEntries(Object.entries(value).map(([key, each]) => [key, withoutParam(each, name)])),
+        ) as T;
+    }
+    return value;
+}
+
 /**
  * What a failed call throws: the signal's reason when it aborted, an ImportError with the source
  * info attached, or the error itself.

package/src/sources/sparql/index.ts

@@ -230,13 +230,28 @@ function write(graphed: Graphed, sink: GraphSink, ctx: SourceContext): void {
     }
 }
 
+/**
+ * Order strings by UTF-16 code units, as `Array.prototype.sort` does with no compare function, so
+ * a content hash never depends on the locale.
+ * @param a - one string
+ * @param b - the other
+ * @returns negative, zero or positive
+ */
+function byCodeUnits(a: string, b: string): number {
+    if (a === b) {
+        return 0;
+    }
+
+    return a < b ? -1 : 1;
+}
+
 /**
  * The content hash of some triples, in any order.
  * @param triples - the triples
  * @returns the SHA-256, hex
  */
 function hashOf(triples: readonly Triple[]): Promise<string> {
-    return sha256Hex(triples.map(canonicalTriple).sort().join("\n")); // NOSONAR(S2871): code-unit order is the point: a locale-free, stable order for the hash
+    return sha256Hex(triples.map(canonicalTriple).sort(byCodeUnits).join("\n"));
 }
 
 /**

package/src/sources/sparql/query.ts

@@ -37,19 +37,73 @@ function bare(query: string): string {
         .replaceAll(/#[^\n]*/g, "");
 }
 
+/** The keywords of SPARQL Update, which a query this source sends never holds. */
+const UPDATE_KEYWORDS = /\b(INSERT|DELETE|LOAD|CLEAR|DROP|CREATE|ADD|MOVE|COPY|WITH)\b/i;
+
+/** One declaration of the prologue, as `bare` leaves it: `BASE <>` or `PREFIX ex: <>`. */
+const DECLARATION = /^\s*(?:BASE|PREFIX\s*[^\s:<]*:)\s*<>/i;
+
+/**
+ * Bare text after its prologue, one declaration at a time.
+ * @param text - the bare text
+ * @returns what follows the prologue
+ */
+function afterPrologue(text: string): string {
+    let rest = text;
+    for (let found = DECLARATION.exec(rest); found !== null; found = DECLARATION.exec(rest)) {
+        rest = rest.slice(found[0].length);
+    }
+    return rest;
+}
+
+/**
+ * The bare text with variables and prefixed names blanked too, so a name such as `ex:LIMIT` or
+ * `?DELETE` is never read as a keyword.
+ * @param text - the bare text after the prologue
+ * @returns the text holding only keywords, punctuation and blanks
+ */
+function keywordsOnly(text: string): string {
+    // one run of name characters at a time: a run holding a colon is a prefixed name
+    return text.replaceAll(/[?$]\w+/g, "?v").replaceAll(/[\w.%:-]+/g, (run) => (run.includes(":") ? "_" : run));
+}
+
+/**
+ * Whether bare text holds a `;` outside every brace: SPARQL Update separates its operations so, and
+ * a query never holds one there.
+ * @param text - the text
+ * @returns true when it does
+ */
+function semicolonOutside(text: string): boolean {
+    let depth = 0;
+    for (const char of text) {
+        if (char === "{") {
+            depth++;
+        } else if (char === "}") {
+            depth--;
+        } else if (char === ";" && depth <= 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 /**
- * Read what paging a caller's query takes.
+ * Read what paging a caller's query takes. Its form is the first keyword after the prologue; a
+ * text that holds an update keyword, or a `;` outside braces, has no form, so it is refused.
  * @param query - the query
  * @returns its form, its own modifiers and its template variables
  */
 export function readQuery(query: string): UserQuery {
-    const text = bare(query);
-    const form = /\b(CONSTRUCT|DESCRIBE|SELECT|ASK)\b/i.exec(text)?.[1]?.toLowerCase() ?? null;
+    const body = afterPrologue(bare(query));
+    const text = keywordsOnly(body);
+    const first = /^\s*([A-Za-z]+)/.exec(text)?.[1]?.toUpperCase() ?? "";
+    const update = UPDATE_KEYWORDS.test(text) || semicolonOutside(text);
+    const form = !update && ["CONSTRUCT", "DESCRIBE", "SELECT", "ASK"].includes(first) ? first.toLowerCase() : null;
     // the solution modifiers follow the last closing brace (a trailing VALUES block is not handled)
     const tail = text.slice(text.lastIndexOf("}") + 1);
     let templateVars: string[] = [];
     if (form === "construct") {
-        const template = /\bCONSTRUCT\s*\{([^}]*)\}/i.exec(text)?.[1] ?? "";
+        const template = /\bCONSTRUCT\s*\{([^}]*)\}/i.exec(body)?.[1] ?? "";
         templateVars = [...new Set(template.match(/[?$]\w+/g) ?? [])].map((v) => `?${v.slice(1)}`);
     }
     return {

package/src/sources/types.ts

@@ -378,6 +378,34 @@ export interface SourceSample {
     readonly firstPage?: number | undefined;
 }
 
+/**
+ * One credential a source takes, as its descriptor declares it: a name a project file records
+ * instead of the credential, the kind, whether a load needs it, and for an API key how it is sent.
+ * `credentials.forSlot(slot, value)` makes the credential it describes. Experimental.
+ * @category Network sources
+ */
+export interface SourceCredentialSlot {
+    /** The slot's name, recorded as a load's `credentialSlot` ("api_key", "default"). */
+    readonly name: string;
+    /** The credential kind it takes. */
+    readonly kind: CredentialKind;
+    /** Whether a load needs it. @defaultValue false */
+    readonly required?: boolean | undefined;
+    /** Its default name; catalog data an application may replace by name. */
+    readonly plainName?: string | undefined;
+    /** For an API key: the query parameter or request header it is sent in. */
+    readonly send?:
+        | {
+              /** The query parameter the key is sent in. */
+              readonly query: string;
+          }
+        | {
+              /** The request header the key is sent in. */
+              readonly header: string;
+          }
+        | undefined;
+}
+
 /**
  * What a form needs to ask for a source's parameters, as data. Only `plainName` and `params` are
  * required. Experimental.
@@ -396,6 +424,11 @@ export interface SourceDescriptor {
     readonly auth?: readonly CredentialKind[] | undefined;
     /** Whether a credential is needed. @defaultValue "none" */
     readonly authRequired?: "required" | "optional" | "none" | undefined;
+    /**
+     * The credential slots it takes. A source that takes a credential (`auth`) and declares none
+     * has one slot, `default`: `credentialSlots(descriptor)` reads either. @defaultValue []
+     */
+    readonly credentials?: readonly SourceCredentialSlot[] | undefined;
     /**
      * Where an `api-key` credential goes, for a form that asks for the key alone: pass it to
      * `credentials.apiKey(descriptor.apiKey, key)`. Absent when the caller must say.

package/src/styles/canon/enums.ts

@@ -231,11 +231,20 @@ const GRAPHVIZ: Partial<Record<StyleEnumKind, Aliases>> = {
     arrow: { "other:curve": ["curve"], "other:inv-curve": ["icurve"] },
 };
 
+/**
+ * yEd's (yFiles for Java's) STANDARD arrow is "a triangle with impressed bottom side", a notched arrow; its DELTA is the
+ * plain triangle (https://docs.yworks.com/yfiles/doc/api/y/view/Arrow.html).
+ */
+const YED: Partial<Record<StyleEnumKind, Aliases>> = {
+    arrow: { vee: ["standard"] },
+};
+
 /** Per-dialect overrides; a dialect not listed reads the common table alone. */
 const OVERRIDES: ReadonlyMap<string, Partial<Record<StyleEnumKind, Aliases>>> = new Map([
     ["vis", VIS],
     ["graphviz", GRAPHVIZ],
     ["cytoscape-js", CYJS],
+    ["yed", YED],
 ]);
 
 const OPEN: ReadonlySet<StyleEnumKind> = new Set<StyleEnumKind>(["shape", "borderStyle", "lineStyle", "arrow"]);